diff --git a/MIGRATION.md b/MIGRATION.md index 7df237086..55317524d 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -99,7 +99,7 @@ Note: Blobfuse2 accepts all CLI parameters that Blobfuse does, but may ignore pa | --log-level=LOG_WARNING | --log-level=LOG_WARNING | logging.level | | | --use-attr-cache=true | --use-attr-cache=true | attr_cache | Add attr_cache to the components list | | --use-adls=false | --use-adls=false | azstorage.type | Specify either 'block' or 'adls' | -| --no-symlinks=false | --no-symlinks=true | attr_cache.no-symlinks | | +| --no-symlinks=false | --no-symlinks=false | attr_cache.no-symlinks | | | --cache-on-list=true | --cache-on-list=true | attr_cache.no-cache-on-list | This parameter has the opposite boolean semantics | | --upload-modified-only=true | --upload-modified-only=true | | Always on in blobfuse2 | | --max-concurrency=12 | --max-concurrency=12 | azstorage.max-concurrency | | diff --git a/README.md b/README.md index e857c9fb3..a17cde655 100755 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ To learn about a specific command, just include the name of the command (For exa * `--lazy-write` : To enable async close file handle call and schedule the upload in background. - Attribute cache options * `--attr-cache-timeout=`: The timeout for the attribute cache entries. - * `--no-symlinks=true`: To improve performance disable symlink support. + * `--no-symlinks=false`: By default symlinks will be supported and the performance overhead, that earlier existed, has been resolved. - Storage options * `--container-name=`: The container to mount. * `--cancel-list-on-mount-seconds=`: Time for which list calls will be blocked after mount. ( prevent billing charges on mounting) diff --git a/azure-pipeline-templates/huge-list-test.yml b/azure-pipeline-templates/huge-list-test.yml index c6f292077..03949d926 100755 --- a/azure-pipeline-templates/huge-list-test.yml +++ b/azure-pipeline-templates/huge-list-test.yml @@ -55,6 +55,10 @@ steps: env: mount_dir: ${{ parameters.mount_dir }} + - script: grep "OUTGOING REQUEST" blobfuse2-logs.txt | wc -l + displayName: 'HugeList: ${{ parameters.idstring }} Request Count' + continueOnError: true + - script: | cat blobfuse2-logs.txt displayName: 'View Logs' diff --git a/component/azstorage/block_blob.go b/component/azstorage/block_blob.go index 4480faedd..36ae82da1 100644 --- a/component/azstorage/block_blob.go +++ b/component/azstorage/block_blob.go @@ -101,9 +101,10 @@ func (bb *BlockBlob) Configure(cfg AzStorageConfig) error { } bb.listDetails = container.ListBlobsInclude{ - Metadata: true, - Deleted: false, - Snapshots: false, + Metadata: true, + Deleted: false, + Snapshots: false, + Permissions: false, //Added to get permissions, acl, group, owner for HNS accounts } return nil @@ -457,6 +458,7 @@ func (bb *BlockBlob) getAttrUsingRest(name string) (attr *internal.ObjAttr, err parseMetadata(attr, prop.Metadata) + // We do not get permissions as part of this getAttr call hence setting the flag to true attr.Flags.Set(internal.PropFlagModeDefault) return attr, nil @@ -534,16 +536,11 @@ func (bb *BlockBlob) List(prefix string, marker *string, count int32) ([]*intern } }(marker)) - blobList := make([]*internal.ObjAttr, 0) - if count == 0 { count = common.MaxDirListCount } - listPath := filepath.Join(bb.Config.prefixPath, prefix) - if (prefix != "" && prefix[len(prefix)-1] == '/') || (prefix == "" && bb.Config.prefixPath != "") { - listPath += "/" - } + listPath := bb.getListPath(prefix) // Get a result segment starting with the blob indicated by the current Marker. pager := bb.Container.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{ @@ -562,46 +559,47 @@ func (bb *BlockBlob) List(prefix string, marker *string, count int32) ([]*intern if err != nil { log.Err("BlockBlob::List : Failed to list the container with the prefix %s", err.Error) - return blobList, nil, err - } - - dereferenceTime := func(input *time.Time, defaultTime time.Time) time.Time { - if input == nil { - return defaultTime - } else { - return *input - } + return nil, nil, err } // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) // Since block blob does not support acls, we set mode to 0 and FlagModeDefault to true so the fuse layer can return the default permission. + blobList, dirList, err := bb.processBlobItems(listBlob.Segment.BlobItems) + if err != nil { + return nil, nil, err + } + + // In case virtual directory exists but its corresponding 0 byte marker file is not there holding hdi_isfolder then just iterating + // over BlobItems will fail to identify that directory. In such cases BlobPrefixes help to list all directories + // dirList contains all dirs for which we got 0 byte meta file in this iteration, so exclude those and add rest to the list + // Note: Since listing is paginated, sometimes the marker file may come in a different iteration from the BlobPrefix. For such + // cases we manually call GetAttr to check the existence of the marker file. + err = bb.processBlobPrefixes(listBlob.Segment.BlobPrefixes, dirList, &blobList) + if err != nil { + return nil, nil, err + } + + return blobList, listBlob.NextMarker, nil +} + +func (bb *BlockBlob) getListPath(prefix string) string { + listPath := filepath.Join(bb.Config.prefixPath, prefix) + if (prefix != "" && prefix[len(prefix)-1] == '/') || (prefix == "" && bb.Config.prefixPath != "") { + listPath += "/" + } + return listPath +} + +func (bb *BlockBlob) processBlobItems(blobItems []*container.BlobItem) ([]*internal.ObjAttr, map[string]bool, error) { + blobList := make([]*internal.ObjAttr, 0) // For some directories 0 byte meta file may not exists so just create a map to figure out such directories - var dirList = make(map[string]bool) - for _, blobInfo := range listBlob.Segment.BlobItems { - attr := &internal.ObjAttr{} - if blobInfo.Properties.CustomerProvidedKeySHA256 != nil && *blobInfo.Properties.CustomerProvidedKeySHA256 != "" { - log.Trace("BlockBlob::List : blob is encrypted with customer provided key so fetching metadata explicitly using REST") - attr, err = bb.getAttrUsingRest(*blobInfo.Name) - if err != nil { - log.Err("BlockBlob::List : Failed to get properties of blob %s", *blobInfo.Name) - return blobList, nil, err - } - } else { - attr = &internal.ObjAttr{ - Path: split(bb.Config.prefixPath, *blobInfo.Name), - Name: filepath.Base(*blobInfo.Name), - Size: *blobInfo.Properties.ContentLength, - Mode: 0, - Mtime: *blobInfo.Properties.LastModified, - Atime: dereferenceTime(blobInfo.Properties.LastAccessedOn, *blobInfo.Properties.LastModified), - Ctime: *blobInfo.Properties.LastModified, - Crtime: dereferenceTime(blobInfo.Properties.CreationTime, *blobInfo.Properties.LastModified), - Flags: internal.NewFileBitMap(), - MD5: blobInfo.Properties.ContentMD5, - } - parseMetadata(attr, blobInfo.Metadata) - attr.Flags.Set(internal.PropFlagModeDefault) + dirList := make(map[string]bool) + + for _, blobInfo := range blobItems { + attr, err := bb.getBlobAttr(blobInfo) + if err != nil { + return nil, nil, err } blobList = append(blobList, attr) @@ -612,35 +610,77 @@ func (bb *BlockBlob) List(prefix string, marker *string, count int32) ([]*intern } } - // In case virtual directory exists but its corresponding 0 byte marker file is not there holding hdi_isfolder then just iterating - // over BlobItems will fail to identify that directory. In such cases BlobPrefixes help to list all directories - // dirList contains all dirs for which we got 0 byte meta file in this iteration, so exclude those and add rest to the list - // Note: Since listing is paginated, sometimes the marker file may come in a different iteration from the BlobPrefix. For such - // cases we manually call GetAttr to check the existence of the marker file. - for _, blobInfo := range listBlob.Segment.BlobPrefixes { + return blobList, dirList, nil +} + +func (bb *BlockBlob) getBlobAttr(blobInfo *container.BlobItem) (*internal.ObjAttr, error) { + if blobInfo.Properties.CustomerProvidedKeySHA256 != nil && *blobInfo.Properties.CustomerProvidedKeySHA256 != "" { + log.Trace("BlockBlob::List : blob is encrypted with customer provided key so fetching metadata explicitly using REST") + return bb.getAttrUsingRest(*blobInfo.Name) + } + mode, err := bb.getFileMode(blobInfo.Properties.Permissions) + if err != nil { + mode = 0 + log.Warn("BlockBlob::getBlobAttr : Failed to get file mode for %s [%s]", *blobInfo.Name, err.Error()) + } + + attr := &internal.ObjAttr{ + Path: split(bb.Config.prefixPath, *blobInfo.Name), + Name: filepath.Base(*blobInfo.Name), + Size: *blobInfo.Properties.ContentLength, + Mode: mode, + Mtime: *blobInfo.Properties.LastModified, + Atime: bb.dereferenceTime(blobInfo.Properties.LastAccessedOn, *blobInfo.Properties.LastModified), + Ctime: *blobInfo.Properties.LastModified, + Crtime: bb.dereferenceTime(blobInfo.Properties.CreationTime, *blobInfo.Properties.LastModified), + Flags: internal.NewFileBitMap(), + MD5: blobInfo.Properties.ContentMD5, + } + + parseMetadata(attr, blobInfo.Metadata) + if !bb.listDetails.Permissions { + // In case of HNS account do not set this flag + attr.Flags.Set(internal.PropFlagModeDefault) + } + + return attr, nil +} + +func (bb *BlockBlob) getFileMode(permissions *string) (os.FileMode, error) { + if permissions == nil { + return 0, nil + } + return getFileMode(*permissions) +} + +func (bb *BlockBlob) dereferenceTime(input *time.Time, defaultTime time.Time) time.Time { + if input == nil { + return defaultTime + } + return *input +} + +func (bb *BlockBlob) processBlobPrefixes(blobPrefixes []*container.BlobPrefix, dirList map[string]bool, blobList *[]*internal.ObjAttr) error { + for _, blobInfo := range blobPrefixes { if _, ok := dirList[*blobInfo.Name]; ok { // marker file found in current iteration, skip adding the directory continue } else { - // marker file not found in current iteration, so we need to manually check attributes via REST - _, err := bb.getAttrUsingRest(*blobInfo.Name) - // marker file also not found via manual check, safe to add to list - if err == syscall.ENOENT { - // For these dirs we get only the name and no other properties so hardcoding time to current time - name := strings.TrimSuffix(*blobInfo.Name, "/") - attr := &internal.ObjAttr{ - Path: split(bb.Config.prefixPath, name), - Name: filepath.Base(name), - Size: 4096, - Mode: os.ModeDir, - Mtime: time.Now(), - Flags: internal.NewDirBitMap(), + //Check to see if its a HNS account and we received properties in blob prefixes + if bb.listDetails.Permissions { + attr, err := bb.createDirAttrWithPermissions(blobInfo) + if err != nil { + return err + } + *blobList = append(*blobList, attr) + } else { + // marker file not found in current iteration, so we need to manually check attributes via REST + _, err := bb.getAttrUsingRest(*blobInfo.Name) + // marker file also not found via manual check, safe to add to list + if err == syscall.ENOENT { + attr := bb.createDirAttr(*blobInfo.Name) + *blobList = append(*blobList, attr) } - attr.Atime = attr.Mtime - attr.Crtime = attr.Mtime - attr.Ctime = attr.Mtime - attr.Flags.Set(internal.PropFlagModeDefault) - blobList = append(blobList, attr) } } } @@ -650,7 +690,54 @@ func (bb *BlockBlob) List(prefix string, marker *string, count int32) ([]*intern delete(dirList, k) } - return blobList, listBlob.NextMarker, nil + return nil +} + +func (bb *BlockBlob) createDirAttr(name string) *internal.ObjAttr { + // For these dirs we get only the name and no other properties so hardcoding time to current time + name = strings.TrimSuffix(name, "/") + attr := &internal.ObjAttr{ + Path: split(bb.Config.prefixPath, name), + Name: filepath.Base(name), + Size: 4096, + Mode: os.ModeDir, + Mtime: time.Now(), + Flags: internal.NewDirBitMap(), + } + attr.Atime = attr.Mtime + attr.Crtime = attr.Mtime + attr.Ctime = attr.Mtime + + // This is called only in case of FNS when blobPrefix is there but the marker does not exists + attr.Flags.Set(internal.PropFlagModeDefault) + return attr +} + +func (bb *BlockBlob) createDirAttrWithPermissions(blobInfo *container.BlobPrefix) (*internal.ObjAttr, error) { + if blobInfo.Properties == nil { + return nil, fmt.Errorf("failed to get properties of blobprefix %s", *blobInfo.Name) + } + + mode, err := bb.getFileMode(blobInfo.Properties.Permissions) + if err != nil { + mode = 0 + log.Warn("BlockBlob::createDirAttrWithPermissions : Failed to get file mode for %s [%s]", *blobInfo.Name, err.Error()) + } + + name := strings.TrimSuffix(*blobInfo.Name, "/") + attr := &internal.ObjAttr{ + Path: split(bb.Config.prefixPath, name), + Name: filepath.Base(name), + Size: *blobInfo.Properties.ContentLength, + Mode: mode, + Mtime: *blobInfo.Properties.LastModified, + Atime: bb.dereferenceTime(blobInfo.Properties.LastAccessedOn, *blobInfo.Properties.LastModified), + Ctime: *blobInfo.Properties.LastModified, + Crtime: bb.dereferenceTime(blobInfo.Properties.CreationTime, *blobInfo.Properties.LastModified), + Flags: internal.NewDirBitMap(), + } + + return attr, nil } // track the progress of download of blobs where every 100MB of data downloaded is being tracked. It also tracks the completion of download diff --git a/component/azstorage/block_blob_test.go b/component/azstorage/block_blob_test.go index 11c3347d9..a5267b705 100644 --- a/component/azstorage/block_blob_test.go +++ b/component/azstorage/block_blob_test.go @@ -3452,6 +3452,47 @@ func (suite *blockBlobTestSuite) UtilityFunctionTruncateFileToLarger(size int, t } +func (s *blockBlobTestSuite) TestList() { + defer s.cleanupTest() + // Setup + s.tearDownTestHelper(false) // Don't delete the generated container. + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.dfs.core.windows.net/\n type: block\n account-key: %s\n mode: key\n container: %s\n fail-unsupported-op: true", + storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockAccount, storageTestConfigurationParameters.BlockKey, s.container) + s.setupTestHelper(config, s.container, true) + + base := generateDirectoryName() + s.setupHierarchy(base) + + blobList, marker, err := s.az.storage.List("", nil, 0) + s.assert.Nil(err) + emptyString := "" + s.assert.Equal(&emptyString, marker) + s.assert.NotNil(blobList) + s.assert.EqualValues(3, len(blobList)) + + // Test listing with prefix + blobList, marker, err = s.az.storage.List(base+"b/", nil, 0) + s.assert.Nil(err) + s.assert.Equal(&emptyString, marker) + s.assert.NotNil(blobList) + s.assert.EqualValues(1, len(blobList)) + s.assert.EqualValues("c1", blobList[0].Name) + + // Test listing with marker + blobList, marker, err = s.az.storage.List(base, to.Ptr("invalid-marker"), 0) + s.assert.NotNil(err) + s.assert.Equal(0, len(blobList)) + s.assert.Nil(marker) + + // Test listing with count + blobList, marker, err = s.az.storage.List("", nil, 1) + s.assert.Nil(err) + s.assert.NotNil(blobList) + s.assert.NotEmpty(marker) + s.assert.EqualValues(1, len(blobList)) + s.assert.EqualValues(base, blobList[0].Path) +} + // In order for 'go test' to run this suite, we need to create // a normal test function and pass our suite to suite.Run func TestBlockBlob(t *testing.T) { diff --git a/component/azstorage/config.go b/component/azstorage/config.go index c4bbd9671..d4701a97e 100644 --- a/component/azstorage/config.go +++ b/component/azstorage/config.go @@ -512,7 +512,7 @@ func ParseAndValidateConfig(az *AzStorage, opt AzStorageOptions) error { log.Crit("ParseAndValidateConfig : Retry Config: retry-count %d, max-timeout %d, backoff-time %d, max-delay %d, preserve-acl: %v", az.stConfig.maxRetries, az.stConfig.maxTimeout, az.stConfig.backoffTime, az.stConfig.maxRetryDelay, az.stConfig.preserveACL) - log.Crit("ParseAndValidateConfig : Telemetry : %s, honour-ACL %v, disable-symlink %v", az.stConfig.telemetry, az.stConfig.honourACL, az.stConfig.disableSymlink) + log.Crit("ParseAndValidateConfig : Telemetry : %s, honour-ACL %v", az.stConfig.telemetry, az.stConfig.honourACL) return nil } @@ -564,16 +564,6 @@ func ParseAndReadDynamicConfig(az *AzStorage, opt AzStorageOptions, reload bool) az.stConfig.honourACL = false } - // by default symlink will be disabled - az.stConfig.disableSymlink = true - - if config.IsSet("attr_cache.no-symlinks") { - err := config.UnmarshalKey("attr_cache.no-symlinks", &az.stConfig.disableSymlink) - if err != nil { - log.Err("ParseAndReadDynamicConfig : Failed to unmarshal attr_cache.no-symlinks") - } - } - // Auth related reconfig switch opt.AuthMode { case "sas": diff --git a/component/azstorage/connection.go b/component/azstorage/connection.go index de73e4cb5..796f1e040 100644 --- a/component/azstorage/connection.go +++ b/component/azstorage/connection.go @@ -73,10 +73,9 @@ type AzStorageConfig struct { maxResultsForList int32 disableCompression bool - telemetry string - honourACL bool - disableSymlink bool - preserveACL bool + telemetry string + honourACL bool + preserveACL bool // CPK related config cpkEnabled bool diff --git a/component/azstorage/datalake.go b/component/azstorage/datalake.go index e23d69d34..5ee88374a 100644 --- a/component/azstorage/datalake.go +++ b/component/azstorage/datalake.go @@ -36,13 +36,11 @@ package azstorage import ( "context" "fmt" - "io/fs" "net/url" "os" "path/filepath" "strings" "syscall" - "time" "github.com/Azure/azure-storage-fuse/v2/common" "github.com/Azure/azure-storage-fuse/v2/common/log" @@ -103,7 +101,13 @@ func (dl *Datalake) Configure(cfg AzStorageConfig) error { EncryptionAlgorithm: to.Ptr(directory.EncryptionAlgorithmTypeAES256), } } - return dl.BlockBlob.Configure(transformConfig(cfg)) + + err := dl.BlockBlob.Configure(transformConfig(cfg)) + + // List call shall always retrieved permissions for HNS accounts + dl.BlockBlob.listDetails.Permissions = true + + return err } // For dynamic config update the config here @@ -424,116 +428,7 @@ func (dl *Datalake) GetAttr(name string) (attr *internal.ObjAttr, err error) { // This fetches the list using a marker so the caller code should handle marker logic // If count=0 - fetch max entries func (dl *Datalake) List(prefix string, marker *string, count int32) ([]*internal.ObjAttr, *string, error) { - log.Trace("Datalake::List : prefix %s, marker %s", prefix, func(marker *string) string { - if marker != nil { - return *marker - } else { - return "" - } - }(marker)) - - pathList := make([]*internal.ObjAttr, 0) - - if count == 0 { - count = common.MaxDirListCount - } - - prefixPath := filepath.Join(dl.Config.prefixPath, prefix) - if prefix != "" && prefix[len(prefix)-1] == '/' { - prefixPath += "/" - } - - // Get a result segment starting with the path indicated by the current Marker. - pager := dl.Filesystem.NewListPathsPager(false, &filesystem.ListPathsOptions{ - Marker: marker, - MaxResults: &count, - Prefix: &prefixPath, - }) - - // Process the paths returned in this result segment (if the segment is empty, the loop body won't execute) - listPath, err := pager.NextPage(context.Background()) - if err != nil { - log.Err("Datalake::List : Failed to validate account with given auth %s", err.Error()) - m := "" - e := storeDatalakeErrToErr(err) - if e == ErrFileNotFound { // TODO: should this be checked for list calls - return pathList, &m, syscall.ENOENT - } else if e == InvalidPermission { - return pathList, &m, syscall.EACCES - } else { - return pathList, &m, err - } - } - - // Process the paths returned in this result segment (if the segment is empty, the loop body won't execute) - for _, pathInfo := range listPath.Paths { - var attr *internal.ObjAttr - var lastModifiedTime time.Time - if dl.Config.disableSymlink { - var mode fs.FileMode - if pathInfo.Permissions != nil { - mode, err = getFileMode(*pathInfo.Permissions) - if err != nil { - log.Err("Datalake::List : Failed to get file mode for %s [%s]", *pathInfo.Name, err.Error()) - m := "" - return pathList, &m, err - } - } else { - // This happens when a blob account is mounted with type:adls - log.Err("Datalake::List : Failed to get file permissions for %s", *pathInfo.Name) - } - - var contentLength int64 = 0 - if pathInfo.ContentLength != nil { - contentLength = *pathInfo.ContentLength - } else { - // This happens when a blob account is mounted with type:adls - log.Err("Datalake::List : Failed to get file length for %s", *pathInfo.Name) - } - - if pathInfo.LastModified != nil { - lastModifiedTime, err = time.Parse(time.RFC1123, *pathInfo.LastModified) - if err != nil { - log.Err("Datalake::List : Failed to get last modified time for %s [%s]", *pathInfo.Name, err.Error()) - } - } - attr = &internal.ObjAttr{ - Path: *pathInfo.Name, - Name: filepath.Base(*pathInfo.Name), - Size: contentLength, - Mode: mode, - Mtime: lastModifiedTime, - Atime: lastModifiedTime, - Ctime: lastModifiedTime, - Crtime: lastModifiedTime, - Flags: internal.NewFileBitMap(), - } - if pathInfo.IsDirectory != nil && *pathInfo.IsDirectory { - attr.Flags = internal.NewDirBitMap() - attr.Mode = attr.Mode | os.ModeDir - } - } else { - attr, err = dl.GetAttr(*pathInfo.Name) - if err != nil { - log.Err("Datalake::List : Failed to get properties for %s [%s]", *pathInfo.Name, err.Error()) - m := "" - return pathList, &m, err - } - } - - // Note: Datalake list paths does not return metadata/properties. - // To account for this and accurately return attributes when needed, - // we have a flag for whether or not metadata has been retrieved. - // If this flag is not set the attribute cache will call get attributes - // to fetch metadata properties. - // Any method that populates the metadata should set the attribute flag. - // Alternatively, if you want Datalake list paths to return metadata/properties as well. - // pass CLI parameter --no-symlinks=false in the mount command. - pathList = append(pathList, attr) - - } - - return pathList, listPath.Continuation, nil + return dl.BlockBlob.List(prefix, marker, count) } // ReadToFile : Download a file to a local file diff --git a/component/azstorage/datalake_test.go b/component/azstorage/datalake_test.go index f48006131..9fa889170 100644 --- a/component/azstorage/datalake_test.go +++ b/component/azstorage/datalake_test.go @@ -458,7 +458,7 @@ func (s *datalakeTestSuite) TestIsDirEmptyError() { empty := s.az.IsDirEmpty(internal.IsDirEmptyOptions{Name: name}) - s.assert.False(empty) // Note: See comment in BlockBlob.List. BlockBlob behaves differently from Datalake + s.assert.True(empty) // Note: See comment in BlockBlob.List. BlockBlob behaves differently from Datalake // Directory should not be in the account dir := s.containerClient.NewDirectoryClient(name) @@ -494,19 +494,20 @@ func (s *datalakeTestSuite) TestReadDirHierarchy() { s.setupHierarchy(base) // ReadDir only reads the first level of the hierarchy + //Using listblob api lists the files before directories so the order is reversed entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: base}) s.assert.Nil(err) s.assert.EqualValues(2, len(entries)) // Check the dir - s.assert.EqualValues(base+"/c1", entries[0].Path) - s.assert.EqualValues("c1", entries[0].Name) - s.assert.True(entries[0].IsDir()) - s.assert.False(entries[0].IsModeDefault()) - // Check the file - s.assert.EqualValues(base+"/c2", entries[1].Path) - s.assert.EqualValues("c2", entries[1].Name) - s.assert.False(entries[1].IsDir()) + s.assert.EqualValues(base+"/c1", entries[1].Path) + s.assert.EqualValues("c1", entries[1].Name) + s.assert.True(entries[1].IsDir()) s.assert.False(entries[1].IsModeDefault()) + // Check the file + s.assert.EqualValues(base+"/c2", entries[0].Path) + s.assert.EqualValues("c2", entries[0].Name) + s.assert.False(entries[0].IsDir()) + s.assert.False(entries[0].IsModeDefault()) } func (s *datalakeTestSuite) TestReadDirRoot() { @@ -524,21 +525,22 @@ func (s *datalakeTestSuite) TestReadDirRoot() { entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: path}) s.assert.Nil(err) s.assert.EqualValues(3, len(entries)) + //Listblob api lists files before directories so the order is reversed // Check the base dir - s.assert.EqualValues(base, entries[0].Path) - s.assert.EqualValues(base, entries[0].Name) - s.assert.True(entries[0].IsDir()) - s.assert.False(entries[0].IsModeDefault()) - // Check the baseb dir - s.assert.EqualValues(base+"b", entries[1].Path) - s.assert.EqualValues(base+"b", entries[1].Name) + s.assert.EqualValues(base, entries[1].Path) + s.assert.EqualValues(base, entries[1].Name) s.assert.True(entries[1].IsDir()) s.assert.False(entries[1].IsModeDefault()) - // Check the basec file - s.assert.EqualValues(base+"c", entries[2].Path) - s.assert.EqualValues(base+"c", entries[2].Name) - s.assert.False(entries[2].IsDir()) + // Check the baseb dir + s.assert.EqualValues(base+"b", entries[2].Path) + s.assert.EqualValues(base+"b", entries[2].Name) + s.assert.True(entries[2].IsDir()) s.assert.False(entries[2].IsModeDefault()) + // Check the basec file + s.assert.EqualValues(base+"c", entries[0].Path) + s.assert.EqualValues(base+"c", entries[0].Name) + s.assert.False(entries[0].IsDir()) + s.assert.False(entries[0].IsModeDefault()) }) } } @@ -573,7 +575,7 @@ func (s *datalakeTestSuite) TestReadDirSubDirPrefixPath() { s.assert.Nil(err) s.assert.EqualValues(1, len(entries)) // Check the dir - s.assert.EqualValues(base+"/c1"+"/gc1", entries[0].Path) + s.assert.EqualValues("c1"+"/gc1", entries[0].Path) s.assert.EqualValues("gc1", entries[0].Name) s.assert.False(entries[0].IsDir()) s.assert.False(entries[0].IsModeDefault()) @@ -586,7 +588,7 @@ func (s *datalakeTestSuite) TestReadDirError() { entries, err := s.az.ReadDir(internal.ReadDirOptions{Name: name}) - s.assert.NotNil(err) // Note: See comment in BlockBlob.List. BlockBlob behaves differently from Datalake + s.assert.Nil(err) // Note: See comment in BlockBlob.List. BlockBlob behaves differently from Datalake s.assert.Empty(entries) // Directory should not be in the account dir := s.containerClient.NewDirectoryClient(name) @@ -2693,6 +2695,50 @@ func (s *datalakeTestSuite) TestPermissionPreservationWithCommit() { s.assert.Contains(acl, "other::rwx") } +func (s *datalakeTestSuite) TestList() { + defer s.cleanupTest() + // Setup + s.tearDownTestHelper(false) // Don't delete the generated container. + config := fmt.Sprintf("azstorage:\n account-name: %s\n endpoint: https://%s.dfs.core.windows.net/\n type: adls\n account-key: %s\n mode: key\n container: %s\n", + storageTestConfigurationParameters.AdlsAccount, storageTestConfigurationParameters.AdlsAccount, storageTestConfigurationParameters.AdlsKey, s.container) + s.setupTestHelper(config, s.container, false) + + base := generateDirectoryName() + s.setupHierarchy(base) + + blobList, marker, err := s.az.storage.List(base, nil, 0) + s.assert.Nil(err) + emptyString := "" + s.assert.Equal(&emptyString, marker) + s.assert.NotNil(blobList) + s.assert.EqualValues(3, len(blobList)) + s.assert.NotEqual(0, blobList[0].Mode) + + // Test listing with prefix + blobList, marker, err = s.az.storage.List(base+"b/", nil, 0) + s.assert.Nil(err) + s.assert.Equal(&emptyString, marker) + s.assert.NotNil(blobList) + s.assert.EqualValues(1, len(blobList)) + s.assert.EqualValues("c1", blobList[0].Name) + s.assert.NotEqual(0, blobList[0].Mode) + + // Test listing with marker + blobList, marker, err = s.az.storage.List(base, to.Ptr("invalid-marker"), 0) + s.assert.NotNil(err) + s.assert.Equal(0, len(blobList)) + s.assert.Nil(marker) + + // Test listing with count + blobList, marker, err = s.az.storage.List("", nil, 1) + s.assert.Nil(err) + s.assert.NotNil(blobList) + s.assert.NotEmpty(marker) + s.assert.EqualValues(1, len(blobList)) + s.assert.EqualValues(base, blobList[0].Path) + s.assert.NotEqual(0, blobList[0].Mode) +} + // func (s *datalakeTestSuite) TestRAGRS() { // defer s.cleanupTest() // // Setup diff --git a/go.mod b/go.mod index d9a7f3bb9..8e7ddc827 100755 --- a/go.mod +++ b/go.mod @@ -5,10 +5,10 @@ go 1.22.0 toolchain go1.23.1 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.1-0.20250111024739-0443c04246ae + github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.3.1-0.20250111024739-0443c04246ae github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda github.com/fsnotify/fsnotify v1.8.0 github.com/golang/mock v1.6.0 @@ -44,17 +44,17 @@ require ( github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sagikazarmark/locafero v0.6.0 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/afero v1.12.0 // indirect github.com/spf13/cast v1.7.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/crypto v0.32.0 // indirect + golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect ) diff --git a/go.sum b/go.sum index bf59ac008..2872e9779 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw= @@ -8,10 +8,10 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xP github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 h1:mlmW46Q0B79I+Aj4azKC6xDMFN9a9SyZWESlGWYXbFs= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0/go.mod h1:PXe2h+LKcWTX9afWdZoHyODqR4fBa5boUM/8uJfZ0Jo= -github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.3.0 h1:K0iyzgmfcq5zLxnD0kndh2G7kejTUZ5xO41IHYGOYVM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.3.0/go.mod h1:CgYxIvUeJo6+7LdnaArwd1Mpk02d9ATikuJviLrxU5E= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.1-0.20250111024739-0443c04246ae h1:xRwsMEg1i7qDQgsWgU7O5wcuAxT5tf3YgMlSppi5RsI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.1-0.20250111024739-0443c04246ae/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI= +github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.3.1-0.20250111024739-0443c04246ae h1:z8sJA1uaujNMOgx2a3JDRUORe1iKBy3HJDBXRGrc/C4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.3.1-0.20250111024739-0443c04246ae/go.mod h1:oU+vqpGSn4J3c46AKAhWw7eHqikQUZuIBULGzDssQE0= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= @@ -79,16 +79,16 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= -github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sevlyar/go-daemon v0.1.6 h1:EUh1MDjEM4BI109Jign0EaknA2izkOyi0LV3ro3QQGs= github.com/sevlyar/go-daemon v0.1.6/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -108,16 +108,16 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 h1:yqrTHse8TCMW1M1ZCP+VAR/l0kKxwaAIqN/il7x4voA= +golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -126,8 +126,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=