Enable azure gov and improve scalability (#896)

* update nugets, enable azure gov and improve scalability * replace tabs with spaces
Azure-Samples · Dec 14, 2020 · 4ff476c · 4ff476c
1 parent 1ea4310
commit 4ff476c
Show file tree

Hide file tree

Showing 16 changed files with 227 additions and 141 deletions.
diff --git a/samples/batch/transcription-enabled-storage/Connector/BatchClient.cs b/samples/batch/transcription-enabled-storage/Connector/BatchClient.cs
@@ -7,6 +7,7 @@ namespace Connector
 {
     using System;
     using System.Collections.Generic;
+    using System.IO;
     using System.Net;
     using System.Net.Http;
     using System.Text;
@@ -105,7 +106,14 @@ private static async Task<Uri> PostAsync(string path, string subscriptionKey, st
 
                     if (!responseMessage.IsSuccessStatusCode)
                     {
-                        var failureMessage = $"Failure: Status Code {responseMessage.StatusCode}, {responseMessage.Content.Headers}";
+                        var failureMessage = $"Failure - Status Code: {responseMessage.StatusCode}";
+
+                        if (responseMessage.Content != null)
+                        {
+                            var body = await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false);
+                            failureMessage += $", Body: {body}";
+                        }
+
                         log.LogInformation(failureMessage);
                         throw new WebException(failureMessage);
                     }
@@ -141,7 +149,14 @@ private static async Task DeleteAsync(string path, string subscriptionKey, TimeS
 
                     if (!responseMessage.IsSuccessStatusCode)
                     {
-                        var failureMessage = $"Failure: Status Code {responseMessage.StatusCode}, {responseMessage.Content.Headers}";
+                        var failureMessage = $"Failure - Status Code: {responseMessage.StatusCode}";
+
+                        if (responseMessage.Content != null)
+                        {
+                            var body = await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false);
+                            failureMessage += $", Body: {body}";
+                        }
+
                         log.LogInformation(failureMessage);
                         throw new WebException(failureMessage);
                     }
@@ -181,6 +196,13 @@ private static async Task<TResponse> GetAsync<TResponse>(string path, string sub
 
                     var failureMessage = $"Failure: Status Code {responseMessage.StatusCode}";
                     log.LogInformation(failureMessage);
+
+                    if (responseMessage.Content != null)
+                    {
+                        var body = await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false);
+                        failureMessage += $", Body: {body}";
+                    }
+
                     throw new WebException(failureMessage);
                 }
                 catch (OperationCanceledException)

diff --git a/samples/batch/transcription-enabled-storage/Connector/Connector.csproj b/samples/batch/transcription-enabled-storage/Connector/Connector.csproj
@@ -18,21 +18,19 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Azure.Storage.Blobs" Version="12.6.0" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.7.0" />
     <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.ServiceBus" Version="4.1.1" />
     <PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="2.9.8">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.CognitiveServices.Speech" Version="1.14.0" />
-    <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.5" />
+    <PackageReference Include="Microsoft.CognitiveServices.Speech" Version="1.11.0" />
     <PackageReference Include="NAudio" Version="1.10.0" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="System.Data.SqlClient" Version="4.8.1" />
-    <PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/samples/batch/transcription-enabled-storage/Connector/Constants.cs b/samples/batch/transcription-enabled-storage/Connector/Constants.cs
@@ -0,0 +1,30 @@
+// <copyright file="Constants.cs" company="Microsoft Corporation">
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE.md file in the project root for full license information.
+// </copyright>
+
+namespace Connector.Constants
+{
+    public static class Constants
+    {
+        public const int MaxRetryLimit = 16;
+
+        public const int DefaultRetryLimit = 4;
+
+        public const int MaxInitialPollingDelayInMinutes = 120;
+
+        public const int DefaultInitialPollingDelayInMinutes = 2;
+
+        public const int DefaultMaxPollingDelayInMinutes = 120;
+
+        public const char Delimiter = ';';
+
+        public const int MaxMessagesPerFunctionExecution = 10000;
+
+        public const int DefaultMessagesPerFunctionExecution = 2000;
+
+        public const int MaxFilesPerTranscriptionJob = 1000;
+
+        public const int DefaultFilesPerTranscriptionJob = 100;
+    }
+}
diff --git a/samples/batch/transcription-enabled-storage/Connector/StorageConnector.cs b/samples/batch/transcription-enabled-storage/Connector/StorageConnector.cs
@@ -8,17 +8,17 @@ namespace Connector
     using System;
     using System.IO;
     using System.Linq;
+    using System.Text;
     using System.Threading.Tasks;
     using System.Web;
     using Azure.Storage;
+    using Azure.Storage.Blobs;
     using Azure.Storage.Sas;
     using Microsoft.Extensions.Logging;
-    using Microsoft.WindowsAzure.Storage;
-    using Microsoft.WindowsAzure.Storage.Blob;
 
     public class StorageConnector
     {
-        private CloudBlobClient CloudBlobClient;
+        private BlobServiceClient BlobServiceClient;
 
         private string AccountName;
 
@@ -31,11 +31,10 @@ public StorageConnector(string storageConnectionString)
                 throw new ArgumentNullException(nameof(storageConnectionString));
             }
 
-            var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
-
             AccountName = GetValueFromConnectionString("AccountName", storageConnectionString);
             AccountKey = GetValueFromConnectionString("AccountKey", storageConnectionString);
-            CloudBlobClient = storageAccount.CreateCloudBlobClient();
+
+            BlobServiceClient = new BlobServiceClient(storageConnectionString);
         }
 
         public static string GetFileNameFromUri(Uri fileUri)
@@ -80,13 +79,13 @@ public static string GetContainerNameFromUri(Uri fileUri)
 
         public static async Task<byte[]> DownloadFileFromSAS(string blobSas)
         {
-            var blob = new CloudBlockBlob(new Uri(blobSas));
+            var blob = new BlobClient(new Uri(blobSas));
 
             byte[] data;
 
             using (var memoryStream = new MemoryStream())
             {
-                await blob.DownloadToStreamAsync(memoryStream).ConfigureAwait(false);
+                await blob.DownloadToAsync(memoryStream).ConfigureAwait(false);
                 data = new byte[memoryStream.Length];
                 memoryStream.Position = 0;
                 memoryStream.Read(data, 0, data.Length);
@@ -99,49 +98,52 @@ public string CreateSas(Uri fileUri)
         {
             var containerName = GetContainerNameFromUri(fileUri);
             var fileName = GetFileNameFromUri(fileUri);
-            var container = CloudBlobClient.GetContainerReference(containerName);
+            var blobContainerClient = BlobServiceClient.GetBlobContainerClient(containerName);
 
             var sasBuilder = new BlobSasBuilder()
             {
-                BlobContainerName = container.Name,
+                BlobContainerName = blobContainerClient.Name,
                 BlobName = fileName,
                 Resource = "b",
             };
 
-            sasBuilder.StartsOn = DateTime.UtcNow;
+            sasBuilder.StartsOn = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10));
             sasBuilder.ExpiresOn = DateTime.UtcNow.AddDays(1);
             sasBuilder.SetPermissions(BlobContainerSasPermissions.Read);
 
             var sasToken = sasBuilder.ToSasQueryParameters(new StorageSharedKeyCredential(AccountName, AccountKey)).ToString();
-            return $"{container.GetBlockBlobReference(fileName).Uri}?{sasToken}";
+            return $"{blobContainerClient.GetBlobClient(fileName).Uri}?{sasToken}";
         }
 
         public async Task WriteTextFileToBlobAsync(string content, string containerName, string fileName, ILogger log)
         {
             log.LogInformation($"Writing file {fileName} to container {containerName}.");
-            var container = CloudBlobClient.GetContainerReference(containerName);
-            var blockBlob = container.GetBlockBlobReference(fileName);
+            var container = BlobServiceClient.GetBlobContainerClient(containerName);
+            var blockBlob = container.GetBlobClient(fileName);
 
-            await blockBlob.UploadTextAsync(content).ConfigureAwait(false);
+            using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(content)))
+            {
+                await blockBlob.UploadAsync(stream).ConfigureAwait(false);
+            }
         }
 
         public async Task MoveFileAsync(string inputContainerName,  string inputFileName, string outputContainerName, string outputFileName, ILogger log)
         {
             log.LogInformation($"Moving file {inputFileName} from container {inputContainerName} to {outputFileName} in container {outputContainerName}.");
-            var inputContainer = CloudBlobClient.GetContainerReference(inputContainerName);
-            var inputBlockBlob = inputContainer.GetBlockBlobReference(inputFileName);
+            var inputContainerClient = BlobServiceClient.GetBlobContainerClient(inputContainerName);
+            var inputBlockBlobClient = inputContainerClient.GetBlobClient(inputFileName);
 
-            if (!await inputBlockBlob.ExistsAsync().ConfigureAwait(false))
+            if (!await inputBlockBlobClient.ExistsAsync().ConfigureAwait(false))
             {
                 log.LogError($"File {inputFileName} does not exist in container {inputContainerName}. Returning.");
                 return;
             }
 
-            var outputContainer = CloudBlobClient.GetContainerReference(outputContainerName);
-            var outputBlockBlob = outputContainer.GetBlockBlobReference(outputFileName);
+            var outputContainerClient = BlobServiceClient.GetBlobContainerClient(outputContainerName);
+            var outputBlockBlobClient = outputContainerClient.GetBlobClient(outputFileName);
 
-            await outputBlockBlob.StartCopyAsync(inputBlockBlob).ConfigureAwait(false);
-            await inputBlockBlob.DeleteAsync().ConfigureAwait(false);
+            await outputBlockBlobClient.StartCopyFromUriAsync(inputBlockBlobClient.Uri).ConfigureAwait(false);
+            await inputBlockBlobClient.DeleteAsync().ConfigureAwait(false);
         }
 
         private static string GetValueFromConnectionString(string key, string connectionString)

diff --git a/samples/batch/transcription-enabled-storage/FetchTranscription/FetchTranscription.csproj b/samples/batch/transcription-enabled-storage/FetchTranscription/FetchTranscription.csproj
@@ -16,18 +16,18 @@
     <OutputPath>bin\Release\</OutputPath>
   </PropertyGroup>  
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.ServiceBus" Version="4.1.1" />
-    <PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="2.9.8">
+    <PackageReference Include="Microsoft.Azure.WebJobs.Extensions.ServiceBus" Version="4.2.0" />
+    <PackageReference Include="System.Data.SqlClient" Version="4.8.2" />
+    <PackageReference Include="Microsoft.CognitiveServices.Speech" Version="1.14.0" />
+    <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.11" />
+    <PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="3.3.1">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.CognitiveServices.Speech" Version="1.14.0" />
-    <PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.5" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="System.Data.SqlClient" Version="4.8.1" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Connector\Connector.csproj" />

diff --git a/...ranscription-enabled-storage/FetchTranscription/FetchTranscriptionEnvironmentVariables.cs b/...ranscription-enabled-storage/FetchTranscription/FetchTranscriptionEnvironmentVariables.cs
@@ -6,6 +6,7 @@
 namespace FetchTranscriptionFunction
 {
     using System;
+    using Connector.Constants;
     using Connector.Enums;
     using Connector.Extensions;
 
@@ -19,6 +20,12 @@ public static class FetchTranscriptionEnvironmentVariables
 
         public static readonly bool UseSqlDatabase = bool.TryParse(Environment.GetEnvironmentVariable(nameof(UseSqlDatabase), EnvironmentVariableTarget.Process), out UseSqlDatabase) && UseSqlDatabase;
 
+        public static readonly int InitialPollingDelayInMinutes = int.TryParse(Environment.GetEnvironmentVariable(nameof(InitialPollingDelayInMinutes), EnvironmentVariableTarget.Process), out InitialPollingDelayInMinutes) ? InitialPollingDelayInMinutes.ClampInt(2, Constants.MaxInitialPollingDelayInMinutes) : Constants.DefaultInitialPollingDelayInMinutes;
+
+        public static readonly int MaxPollingDelayInMinutes = int.TryParse(Environment.GetEnvironmentVariable(nameof(MaxPollingDelayInMinutes), EnvironmentVariableTarget.Process), out MaxPollingDelayInMinutes) ? MaxPollingDelayInMinutes : Constants.DefaultMaxPollingDelayInMinutes;
+
+        public static readonly int RetryLimit = int.TryParse(Environment.GetEnvironmentVariable(nameof(RetryLimit), EnvironmentVariableTarget.Process), out RetryLimit) ? RetryLimit.ClampInt(1, Constants.MaxRetryLimit) : Constants.DefaultRetryLimit;
+
         public static readonly string AudioInputContainer = Environment.GetEnvironmentVariable(nameof(AudioInputContainer), EnvironmentVariableTarget.Process);
 
         public static readonly string AzureSpeechServicesKey = Environment.GetEnvironmentVariable(nameof(AzureSpeechServicesKey), EnvironmentVariableTarget.Process);
@@ -37,16 +44,10 @@ public static class FetchTranscriptionEnvironmentVariables
 
         public static readonly string JsonResultOutputContainer = Environment.GetEnvironmentVariable(nameof(JsonResultOutputContainer), EnvironmentVariableTarget.Process);
 
-        public static readonly int RetryLimit = int.TryParse(Environment.GetEnvironmentVariable(nameof(RetryLimit), EnvironmentVariableTarget.Process), out RetryLimit) ? RetryLimit.ClampInt(1, MaxRetryLimit) : DefaultRetryLimit;
-
         public static readonly string StartTranscriptionServiceBusConnectionString = Environment.GetEnvironmentVariable(nameof(StartTranscriptionServiceBusConnectionString), EnvironmentVariableTarget.Process);
 
         public static readonly string TextAnalyticsKey = Environment.GetEnvironmentVariable(nameof(TextAnalyticsKey), EnvironmentVariableTarget.Process);
 
         public static readonly string TextAnalyticsRegion = Environment.GetEnvironmentVariable(nameof(TextAnalyticsRegion), EnvironmentVariableTarget.Process);
-
-        private const int MaxRetryLimit = 16;
-
-        private const int DefaultRetryLimit = 4;
     }
 }
diff --git a/samples/batch/transcription-enabled-storage/FetchTranscription/TextAnalytics.cs b/samples/batch/transcription-enabled-storage/FetchTranscription/TextAnalytics.cs
@@ -202,8 +202,8 @@ private static List<TextAnalyticsEntity> RemoveOverlappingEntities(List<TextAnal
                 return textAnalyticsEntities;
             }
 
-            textAnalyticsEntities.OrderBy(o => o.Offset);
-            List<TextAnalyticsEntity> resultEntities = textAnalyticsEntities;
+            var orderedEntities = textAnalyticsEntities.OrderBy(o => o.Offset);
+            var resultEntities = orderedEntities.ToList();
             bool foundOverlap;
 
             do