Skip to content
This repository has been archived by the owner on Aug 4, 2022. It is now read-only.

Commit

Permalink
address PR comments
Browse files Browse the repository at this point in the history
  • Loading branch information
dwaijam committed Jun 30, 2018
1 parent 272feaa commit e7f44e3
Show file tree
Hide file tree
Showing 10 changed files with 99 additions and 57 deletions.
60 changes: 35 additions & 25 deletions lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs
Original file line number Diff line number Diff line change
Expand Up @@ -69,26 +69,22 @@ public void Dispose()

private bool CheckBlobExists(ICloudBlob blob)
{
var task = blob.ExistsAsync();
return task.Result;
return blob.ExistsAsync().GetAwaiter().GetResult();
}

private bool CheckContainerExists(CloudBlobContainer container)
{
var task = container.ExistsAsync();
return task.Result;
return container.ExistsAsync().GetAwaiter().GetResult();
}

private ICloudBlob GetBlobReferenceFromServer(CloudBlobContainer container, string blobName)
{
var task = container.GetBlobReferenceFromServerAsync(blobName);
return task.Result;
return container.GetBlobReferenceFromServerAsync(blobName).GetAwaiter().GetResult();
}

private string DownloadText(CloudBlockBlob blob)
{
var task = blob.DownloadTextAsync();
return task.Result;
return blob.DownloadTextAsync().GetAwaiter().GetResult();
}

[Fact(Skip = SkipMessage)]
Expand Down Expand Up @@ -268,18 +264,34 @@ public void TestCopyFromLocalE2E()
}

[Fact(Skip = SkipMessage)]
public void TestIsDirectory()
public void TestIsDirectoryValidDirectoryLevel1E2E()
{
const string Directory1 = "dir1";
const string Directory2 = "dir1/dir2";
const string Directory3 = "dir3";
var blockBlobs1 = CreateTempBlobs(Directory1);
CreateTempBlobs(Directory2);
const string Directory = "dir";
CreateTempBlobs(Directory);
Assert.True(_fileSystem.IsDirectory(PathToFile(Directory)));
}

Assert.True(_fileSystem.IsDirectory(PathToFile(Directory1)));
Assert.True(_fileSystem.IsDirectory(PathToFile(Directory2)));
Assert.False(_fileSystem.IsDirectory(PathToFile(Directory3)));
Assert.False(_fileSystem.IsDirectory(blockBlobs1.First().Uri));
[Fact(Skip = SkipMessage)]
public void TestIsDirectoryValidDirectoryLevel2E2E()
{
const string Directory = "dir1/dir2";
CreateTempBlobs(Directory);
Assert.True(_fileSystem.IsDirectory(PathToFile(Directory)));
}

[Fact(Skip = SkipMessage)]
public void TestIsDirectoryFakeDirectoryE2E()
{
const string Directory = "dir";
Assert.False(_fileSystem.IsDirectory(PathToFile(Directory)));
}

[Fact(Skip = SkipMessage)]
public void TestIsDirectoryFileE2E()
{
const string Directory = "dir";
var blockBlobs = CreateTempBlobs(Directory);
Assert.False(_fileSystem.IsDirectory(blockBlobs.First().Uri));
}

[Fact(Skip = SkipMessage)]
Expand Down Expand Up @@ -328,18 +340,16 @@ public void TestDeleteDirectorySecondLevelE2E()
Assert.True(CheckContainerExists(_container));
}

private List<CloudBlockBlob> CreateTempBlobs(string directory, int fileCount = 3)
private IEnumerable<CloudBlockBlob> CreateTempBlobs(string directory, int fileCount = 3)
{
var blockBlobs = new List<CloudBlockBlob>();
for (var i = 0; i < fileCount; i++)
return Enumerable.Range(0, fileCount).Select(i =>
{
var filePath = directory + '/' + i;
var blockBlob = _container.GetBlockBlobReference(filePath);
UploadFromString(blockBlob, "hello");
Assert.True(CheckBlobExists(blockBlob));
blockBlobs.Add(blockBlob);
}
return blockBlobs;
Assert.True(CheckBlobExists(blockBlob), "Blob does not exist: " + filePath);
return blockBlob;
});
}

private static void UploadFromString(ICloudBlob blob, string str)
Expand Down
18 changes: 13 additions & 5 deletions lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs
Original file line number Diff line number Diff line change
Expand Up @@ -195,16 +195,24 @@ public void TestCreateDirectoryE2E()
}

[Fact(Skip = SkipMessage)]
public void TestIsDirectoryE2E()
public void TestIsDirectoryValidDirectoryE2E()
{
string dirName = $"/{_defaultFolderName}";
string fakeDirName = $"/fakeDir";

_adlsClient.CreateDirectory(dirName);
string fileName = UploadFromString(ContentsText);
Assert.True(_fileSystem.IsDirectory(PathToFile(dirName)));
}

Assert.True(_fileSystem.IsDirectory(PathToFile(dirName)));
[Fact(Skip = SkipMessage)]
public void TestIsDirectoryFakeDirectoryE2E()
{
string fakeDirName = $"/fakeDir";
Assert.False(_fileSystem.IsDirectory(PathToFile(fakeDirName)));
}

[Fact(Skip = SkipMessage)]
public void TestIsDirectoryFileE2E()
{
string fileName = UploadFromString(ContentsText);
Assert.False(_fileSystem.IsDirectory(PathToFile(fileName)));
}

Expand Down
28 changes: 23 additions & 5 deletions lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs
Original file line number Diff line number Diff line change
Expand Up @@ -165,29 +165,47 @@ public void CreateUriForPathWithPrefix()
}

/// <summary>
/// Tests whether .IsDirectory() works.
/// Tests whether .IsDirectory() works for valid directory.
/// </summary>
[Fact(Skip = SkipMessage)]
public void TestIsDirectory()
public void TestIsDirectoryValidDirectory()
{
// Create directory
var remoteDirUri = GetTempUri();
_fileSystem.CreateDirectory(remoteDirUri);

Assert.True(_fileSystem.IsDirectory(remoteDirUri));

// Clean up
_fileSystem.DeleteDirectory(remoteDirUri);
}

/// <summary>
/// Tests whether .IsDirectory() works for fake directory.
/// </summary>
[Fact(Skip = SkipMessage)]
public void TestIsDirectoryFakeDirectory()
{
// Create fake directory uri
var remoteFakeuri = GetTempUri();

Assert.False(_fileSystem.IsDirectory(remoteFakeuri));
}

/// <summary>
/// Tests whether .IsDirectory() works for file.
/// </summary>
[Fact(Skip = SkipMessage)]
public void TestIsDirectoryFile()
{
// Create file
var remoteFileUri = GetTempUri();
var localFile = FileSystemTestUtilities.MakeLocalTempFile();
_fileSystem.CopyFromLocal(localFile, remoteFileUri);

Assert.True(_fileSystem.IsDirectory(remoteDirUri));
Assert.False(_fileSystem.IsDirectory(remoteFakeuri));
Assert.False(_fileSystem.IsDirectory(remoteFileUri));

// Clean up
_fileSystem.DeleteDirectory(remoteDirUri);
_fileSystem.Delete(remoteFileUri);
File.Delete(localFile);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,15 @@ public void CreateDirectory(Uri directoryUri)
public bool IsDirectory(Uri uri)
{
var path = uri.AbsolutePath.TrimStart('/');
const int maxBlobResults = 1;
var blobItems = _client.ListBlobsSegmented(path, false, BlobListingDetails.Metadata, maxBlobResults, null, null, null).Results;
return blobItems.Any() && blobItems.First() is CloudBlobDirectory;
var blobItems = _client.ListBlobsSegmented(
path,
useFlatListing: false,
BlobListingDetails.Metadata,
maxResults: 1,
continuationToken: null,
blobRequestOptions: null,
operationContext: null).Results;
return blobItems.OfType<CloudBlobDirectory>().Any();
}

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,7 @@ public Uri BaseUri

public ICloudBlob GetBlobReferenceFromServer(Uri blobUri)
{
var task = _client.GetBlobReferenceFromServerAsync(blobUri);
return task.Result;
return _client.GetBlobReferenceFromServerAsync(blobUri).GetAwaiter().GetResult();
}

public ICloudBlobContainer GetContainerReference(string containerName)
Expand All @@ -78,15 +77,14 @@ public BlobResultSegment ListBlobsSegmented(
BlobRequestOptions blobRequestOptions,
OperationContext operationContext)
{
var task = _client.ListBlobsSegmentedAsync(
return _client.ListBlobsSegmentedAsync(
prefix,
useFlatListing,
blobListingDetails,
maxResults,
continuationToken,
blobRequestOptions,
operationContext);
return task.Result;
operationContext).GetAwaiter().GetResult();
}

public BlobResultSegment ListDirectoryBlobsSegmented(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,7 @@ public AzureCloudBlobContainer(CloudBlobContainer container, BlobRequestOptions

public bool CreateIfNotExists()
{
var task = _container.CreateIfNotExistsAsync(_requestOptions, null);
return task.Result;
return _container.CreateIfNotExistsAsync(_requestOptions, null).GetAwaiter().GetResult();
}

public void DeleteIfExists()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,13 @@ public ICloudBlobDirectory GetDirectoryReference(string directoryName)

public IEnumerable<IListBlobItem> ListBlobs(bool useFlatListing = false)
{
var task = _directory.ListBlobsSegmentedAsync(useFlatListing, BlobListingDetails.All, null, null, null, null);
return task.Result.Results;
return _directory.ListBlobsSegmentedAsync(
useFlatBlobListing: useFlatListing,
BlobListingDetails.All,
maxResults: null,
null,
null,
null).GetAwaiter().GetResult().Results;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public Stream Open()
{
#if REEF_DOTNET_BUILD
var task = _blob.OpenReadAsync(null, _requestOptions, null);
return task.Result;
return task.GetAwaiter().GetResult();
#else
return _blob.OpenRead(null, _requestOptions, null);
#endif
Expand All @@ -76,17 +76,15 @@ public Stream Open()
public Stream Create()
{
#if REEF_DOTNET_BUILD
var task = _blob.OpenWriteAsync(null, _requestOptions, null);
return task.Result;
return _blob.OpenWriteAsync(null, _requestOptions, null).GetAwaiter().GetResult();
#else
return _blob.OpenWrite(null, _requestOptions, null);
#endif
}

public bool Exists()
{
var task = _blob.ExistsAsync(_requestOptions, null);
return task.Result;
return _blob.ExistsAsync(_requestOptions, null).GetAwaiter().GetResult();
}

public void Delete()
Expand All @@ -101,8 +99,7 @@ public void DeleteIfExists()

public string StartCopy(Uri source)
{
var task = _blob.StartCopyAsync(source, null, null, _requestOptions, null);
return task.Result;
return _blob.StartCopyAsync(source, null, null, _requestOptions, null).GetAwaiter().GetResult();
}

public void DownloadToFile(string path, FileMode mode)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ internal interface ICloudBlobClient
/// Paginates a blob listing with prefix.
/// </summary>
BlobResultSegment ListBlobsSegmented(
string path,
string prefix,
bool useFlatListing,
BlobListingDetails blobListingDetails,
int? maxResults,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
Expand Down Expand Up @@ -249,10 +250,10 @@ public Uri CreateUriForPath(string path)
}
catch (UriFormatException)
{
resultUri = new Uri(new Uri(this.UriPrefix), path);
resultUri = new Uri(new Uri(UriPrefix), path);
}

if (!resultUri.AbsoluteUri.StartsWith(this.UriPrefix))
if (!resultUri.AbsoluteUri.StartsWith(UriPrefix, true, CultureInfo.InvariantCulture))
{
throw new ArgumentException($"Given URI must begin with valid prefix ({this.UriPrefix})", nameof(path));
}
Expand Down

0 comments on commit e7f44e3

Please sign in to comment.