From eb3074a3e4bc9cc9c1e5cb15109c36043cbb0fec Mon Sep 17 00:00:00 2001 From: Dan Lepow Date: Thu, 6 Sep 2018 11:30:59 -0700 Subject: [PATCH 1/9] Updates in progress --- .gitignore | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 940794e..1cf8ba4 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,7 @@ dlldata.c project.lock.json project.fragment.lock.json artifacts/ -**/Properties/launchSettings.json +# **/Properties/launchSettings.json *_i.c *_p.c diff --git a/README.md b/README.md index 76a34ca..2aeeb05 100644 --- a/README.md +++ b/README.md @@ -14,14 +14,14 @@ For details and explanation, see the accompanying article [Run a parallel worklo ## Prerequisites - Azure Batch account and linked general-purpose Azure Storage account -- Visual Studio 2017 +- Visual Studio 2017, or .NET core 2.x for [Linux](https://docs.microsoft.com/dotnet/core/linux-prerequisites?tabs=netcore2x) or [Windows](https://docs.microsoft.com/dotnet/core/windows-prerequisites?tabs=netcore2x) - Windows 64-bit version of [ffmpeg 3.4](https://ffmpeg.zeranoe.com/builds/win64/static/ffmpeg-3.4-win64-static.zip) - Add ffmpeg as an [application package](https://docs.microsoft.com/azure/batch/batch-application-packages) to your Batch account (Application Id: *ffmpeg*, Version: *3.4*) ## Resources - [Azure Batch documentation](https://docs.microsoft.com/azure/batch/) -- [Azure Batch code samples repo](https://github.com/Azure/azure-batch-samples) +- [Azure Batch code samples repo](https://github.com/Azure-Samples/azure-batch-samples) ## Project code of conduct From 271855f4930996ef86dd67bc4f037620d0c6a28a Mon Sep 17 00:00:00 2001 From: Dan Lepow Date: Thu, 6 Sep 2018 12:31:57 -0700 Subject: [PATCH 2/9] Updates to target .NET core 2.1 --- .../BatchDotnetTutorialFfmpeg.csproj | 6 +- .../BatchDotnetTutorialFfmpeg.sln | 5 +- BatchDotnetTutorialFfmpeg/Program.cs | 206 +++++++++--------- .../Properties/launchSettings.json | 8 + README.md | 3 +- 5 files changed, 123 insertions(+), 105 deletions(-) create mode 100644 BatchDotnetTutorialFfmpeg/Properties/launchSettings.json diff --git a/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.csproj b/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.csproj index 4fec3d3..e613d28 100644 --- a/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.csproj +++ b/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.csproj @@ -2,13 +2,13 @@ Exe - net462 + netcoreapp2.1 Microsoft.Azure.Batch.Samples.BatchDotNetTutorialFfmpeg - - + + \ No newline at end of file diff --git a/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.sln b/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.sln index 3c1bdb5..98184b4 100644 --- a/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.sln +++ b/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.sln @@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 VisualStudioVersion = 15.0.26228.0 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BatchDotnetTutorialFfmpeg", "BatchDotnetTutorialFfmpeg.csproj", "{C8DFEFA7-5BC3-4217-88DC-791020D6A909}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BatchDotnetTutorialFfmpeg", "BatchDotnetTutorialFfmpeg.csproj", "{C8DFEFA7-5BC3-4217-88DC-791020D6A909}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{5CAC1578-E7D4-416E-BB9D-1A9800B0F1A6}" EndProject @@ -23,4 +23,7 @@ Global GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {40550F11-FA23-4798-B91D-D045F41F982C} + EndGlobalSection EndGlobal diff --git a/BatchDotnetTutorialFfmpeg/Program.cs b/BatchDotnetTutorialFfmpeg/Program.cs index f2db2d3..137fdbd 100644 --- a/BatchDotnetTutorialFfmpeg/Program.cs +++ b/BatchDotnetTutorialFfmpeg/Program.cs @@ -7,6 +7,7 @@ namespace BatchDotnetTutorialFfmpeg using System.Collections.Generic; using System.Diagnostics; using System.IO; + using System.Threading.Tasks; using Microsoft.Azure.Batch; using Microsoft.Azure.Batch.Auth; using Microsoft.Azure.Batch.Common; @@ -44,119 +45,133 @@ public class Program public static void Main(string[] args) { - if (String.IsNullOrEmpty(BatchAccountName) || - String.IsNullOrEmpty(BatchAccountKey) || - String.IsNullOrEmpty(BatchAccountUrl) || - String.IsNullOrEmpty(StorageAccountName) || - String.IsNullOrEmpty(StorageAccountKey)) + if (String.IsNullOrEmpty(BatchAccountName) || String.IsNullOrEmpty(BatchAccountKey) || String.IsNullOrEmpty(BatchAccountUrl) || + String.IsNullOrEmpty(StorageAccountName) || String.IsNullOrEmpty(StorageAccountKey)) { throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified."); } try { - // START TIMER - Console.WriteLine("Sample start: {0}", DateTime.Now); + // Call the asynchronous version of the Main() method. This is done so that we can await various + // calls to async methods within the "Main" method of this console application. + MainAsync().Wait(); + } + catch (AggregateException) + { + Console.WriteLine(); + Console.WriteLine("One or more exceptions occurred."); Console.WriteLine(); - Stopwatch timer = new Stopwatch(); - timer.Start(); + } + finally + { + Console.WriteLine(); + Console.WriteLine("Sample complete, hit ENTER to exit..."); + Console.ReadLine(); + } + } - // STORAGE SETUP - // Construct the Storage account connection string - string storageConnectionString = String.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", - StorageAccountName, StorageAccountKey); + /// + /// Provides an asynchronous version of the Main method, allowing for the awaiting of async method calls within. + /// + /// A object that represents the asynchronous operation. + private static async Task MainAsync() + { + Console.WriteLine("Sample start: {0}", DateTime.Now); + Console.WriteLine(); + Stopwatch timer = new Stopwatch(); + timer.Start(); + + // Construct the Storage account connection string + string storageConnectionString = + $"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey}"; + + // Retrieve the storage account + CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); + + // Create the blob client, for use in obtaining references to blob storage containers + CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); - // Retrieve the storage account - CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); + + // Use the blob client to create the containers in blob storage + const string inputContainerName = "input"; + const string outputContainerName = "output"; - // Create the blob client, which will be used to obtain references to blob storage containers - CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); + await CreateContainerIfNotExistAsync(blobClient, inputContainerName); + await CreateContainerIfNotExistAsync(blobClient, outputContainerName); - // Use the blob client to create the containers in blob storage - const string inputContainerName = "input"; - const string outputContainerName = "output"; + // RESOURCE FILE SETUP + // Input files: Specify the location of the data files that the tasks process, and + // put them in a List collection. Make sure you have copied the data files to: + // \\InputFiles. - CreateContainerIfNotExist(blobClient, inputContainerName); - CreateContainerIfNotExist(blobClient, outputContainerName); - - // RESOURCE FILE SETUP - // Input files: Specify the location of the data files that the tasks process, and - // put them in a List collection. Make sure you have copied the data files to: - // \\InputFiles. + string inputPath = Path.Combine(Environment.CurrentDirectory, "InputFiles"); - List inputFilePaths = new List(Directory.GetFileSystemEntries(@"..\..\..\InputFiles", "*.mp4", + List inputFilePaths = new List(Directory.GetFileSystemEntries(inputPath, "*.mp4", SearchOption.TopDirectoryOnly)); - // Upload data files. - // Upload the data files using UploadResourceFilesToContainer(). This data will be - // processed by each of the tasks that are executed on the compute nodes within the pool. - List inputFiles = UploadResourceFilesToContainer(blobClient, inputContainerName, inputFilePaths); + // Upload data files. + // Upload the data files using UploadResourceFilesToContainer(). This data will be + // processed by each of the tasks that are executed on the compute nodes within the pool. + List inputFiles = await UploadFilesToContainerAsync(blobClient, inputContainerName, inputFilePaths); - // Obtain a shared access signature that provides write access to the output container to which - // the tasks will upload their output. - string outputContainerSasUrl = GetContainerSasUrl(blobClient, outputContainerName, SharedAccessBlobPermissions.Write); + // Obtain a shared access signature that provides write access to the output container to which + // the tasks will upload their output. + string outputContainerSasUrl = GetContainerSasUrl(blobClient, outputContainerName, SharedAccessBlobPermissions.Write); - // CREATE BATCH CLIENT / CREATE POOL / CREATE JOB / ADD TASKS + // CREATE BATCH CLIENT / CREATE POOL / CREATE JOB / ADD TASKS - // Create a Batch client and authenticate with shared key credentials. - // The Batch client allows the app to interact with the Batch service. - BatchSharedKeyCredentials sharedKeyCredentials = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey); + // Create a Batch client and authenticate with shared key credentials. + // The Batch client allows the app to interact with the Batch service. + BatchSharedKeyCredentials sharedKeyCredentials = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey); - using (BatchClient batchClient = BatchClient.Open(sharedKeyCredentials)) - { - // Create the Batch pool, which contains the compute nodes that execute the tasks. - CreatePoolIfNotExist(batchClient, PoolId); + using (BatchClient batchClient = BatchClient.Open(sharedKeyCredentials)) + { + // Create the Batch pool, which contains the compute nodes that execute the tasks. + CreatePoolIfNotExist(batchClient, PoolId); - // Create the job that runs the tasks. - CreateJobIfNotExist(batchClient, JobId, PoolId); + // Create the job that runs the tasks. + CreateJobIfNotExist(batchClient, JobId, PoolId); - // Create a collection of tasks and add them to the Batch job. - // Provide a shared access signature for the tasks so that they can upload their output - // to the Storage container. - AddTasks(batchClient, JobId, inputFiles, outputContainerSasUrl); + // Create a collection of tasks and add them to the Batch job. + // Provide a shared access signature for the tasks so that they can upload their output + // to the Storage container. + AddTasks(batchClient, JobId, inputFiles, outputContainerSasUrl); - // Monitor task success or failure, specifying a maximum amount of time to wait for - // the tasks to complete. - MonitorTasks(batchClient, JobId, TimeSpan.FromMinutes(30)); + // Monitor task success or failure, specifying a maximum amount of time to wait for + // the tasks to complete. + MonitorTasks(batchClient, JobId, TimeSpan.FromMinutes(30)); - // Delete input container in storage - Console.WriteLine("Deleting container [{0}]...", inputContainerName); - CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName); - container.DeleteIfExists(); + // Delete input container in storage + Console.WriteLine("Deleting container [{0}]...", inputContainerName); + CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName); + await container.DeleteIfExistsAsync(); - // Print out timing info - timer.Stop(); - Console.WriteLine(); - Console.WriteLine("Sample end: {0}", DateTime.Now); - Console.WriteLine("Elapsed time: {0}", timer.Elapsed); - - // Clean up Batch resources (if the user so chooses) - Console.WriteLine(); - Console.Write("Delete job? [yes] no: "); - string response = Console.ReadLine().ToLower(); - if (response != "n" && response != "no") - { - batchClient.JobOperations.DeleteJob(JobId); - } + // Print out timing info + timer.Stop(); + Console.WriteLine(); + Console.WriteLine("Sample end: {0}", DateTime.Now); + Console.WriteLine("Elapsed time: {0}", timer.Elapsed); - Console.Write("Delete pool? [yes] no: "); - response = Console.ReadLine().ToLower(); - if (response != "n" && response != "no") - { - batchClient.PoolOperations.DeletePool(PoolId); - } - } - } - finally - { + // Clean up Batch resources (if the user so chooses) Console.WriteLine(); - Console.WriteLine("Sample complete, hit ENTER to exit..."); - Console.ReadLine(); + Console.Write("Delete job? [yes] no: "); + string response = Console.ReadLine().ToLower(); + if (response != "n" && response != "no") + { + batchClient.JobOperations.DeleteJob(JobId); + } + + Console.Write("Delete pool? [yes] no: "); + response = Console.ReadLine().ToLower(); + if (response != "n" && response != "no") + { + batchClient.PoolOperations.DeletePool(PoolId); + } } } - - + // FUNCTION IMPLEMENTATIONS /// @@ -164,19 +179,12 @@ public static void Main(string[] args) /// /// A . /// The name for the new container. - - private static void CreateContainerIfNotExist(CloudBlobClient blobClient, string containerName) + + private static async Task CreateContainerIfNotExistAsync(CloudBlobClient blobClient, string containerName) { CloudBlobContainer container = blobClient.GetContainerReference(containerName); - - if (container.CreateIfNotExists()) - { - Console.WriteLine("Container [{0}] created.", containerName); - } - else - { - Console.WriteLine("Container [{0}] exists, skipping creation.", containerName); - } + await container.CreateIfNotExistsAsync(); + Console.WriteLine("Creating container [{0}].", containerName); } @@ -189,13 +197,13 @@ private static void CreateContainerIfNotExist(CloudBlobClient blobClient, string /// Name of the blob storage container to which the files are uploaded. /// A collection of paths of the files to be uploaded to the container. /// A collection of objects. - private static List UploadResourceFilesToContainer(CloudBlobClient blobClient, string containerName, List filePaths) + private static async Task> UploadFilesToContainerAsync(CloudBlobClient blobClient, string inputContainerName, List filePaths) { List resourceFiles = new List(); foreach (string filePath in filePaths) { - resourceFiles.Add(UploadResourceFileToContainer(blobClient, containerName, filePath)); + resourceFiles.Add(await UploadResourceFileToContainerAsync(blobClient, inputContainerName, filePath)); } return resourceFiles; @@ -208,7 +216,7 @@ private static List UploadResourceFilesToContainer(CloudBlobClient /// The name of the blob storage container to which the file should be uploaded. /// The full path to the file to upload to Storage. /// A ResourceFile object representing the file in blob storage. - private static ResourceFile UploadResourceFileToContainer(CloudBlobClient blobClient, string containerName, string filePath) + private static async Task UploadResourceFileToContainerAsync(CloudBlobClient blobClient, string containerName, string filePath) { Console.WriteLine("Uploading file {0} to container [{1}]...", filePath, containerName); @@ -217,7 +225,7 @@ private static ResourceFile UploadResourceFileToContainer(CloudBlobClient blobCl CloudBlobContainer container = blobClient.GetContainerReference(containerName); CloudBlockBlob blobData = container.GetBlockBlobReference(blobName); - blobData.UploadFromFile(filePath); + await blobData.UploadFromFileAsync(filePath); // Set the expiry time and permissions for the blob shared access signature. In this case, no start time is specified, // so the shared access signature becomes valid immediately diff --git a/BatchDotnetTutorialFfmpeg/Properties/launchSettings.json b/BatchDotnetTutorialFfmpeg/Properties/launchSettings.json new file mode 100644 index 0000000..4e92bea --- /dev/null +++ b/BatchDotnetTutorialFfmpeg/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "BatchDotnetTutorialFfmpeg": { + "commandName": "Project", + "workingDirectory": "$(ProjectDir)" + } + } +} \ No newline at end of file diff --git a/README.md b/README.md index 2aeeb05..ebee37a 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,7 @@ For details and explanation, see the accompanying article [Run a parallel worklo ## Prerequisites - Azure Batch account and linked general-purpose Azure Storage account -- Visual Studio 2017, or .NET core 2.x for [Linux](https://docs.microsoft.com/dotnet/core/linux-prerequisites?tabs=netcore2x) or [Windows](https://docs.microsoft.com/dotnet/core/windows-prerequisites?tabs=netcore2x) -- Windows 64-bit version of [ffmpeg 3.4](https://ffmpeg.zeranoe.com/builds/win64/static/ffmpeg-3.4-win64-static.zip) +- Visual Studio 2017, or [.NET core 2.1](https://www.microsoft.com/net/download/dotnet-core/2.1) for Linux, macOS, or Windows - Add ffmpeg as an [application package](https://docs.microsoft.com/azure/batch/batch-application-packages) to your Batch account (Application Id: *ffmpeg*, Version: *3.4*) ## Resources From 1c62999aef403f156ad58287633f0c3771736733 Mon Sep 17 00:00:00 2001 From: Dan Lepow Date: Thu, 6 Sep 2018 12:46:27 -0700 Subject: [PATCH 3/9] format change --- BatchDotnetTutorialFfmpeg/Program.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/BatchDotnetTutorialFfmpeg/Program.cs b/BatchDotnetTutorialFfmpeg/Program.cs index 137fdbd..6f197af 100644 --- a/BatchDotnetTutorialFfmpeg/Program.cs +++ b/BatchDotnetTutorialFfmpeg/Program.cs @@ -45,8 +45,11 @@ public class Program public static void Main(string[] args) { - if (String.IsNullOrEmpty(BatchAccountName) || String.IsNullOrEmpty(BatchAccountKey) || String.IsNullOrEmpty(BatchAccountUrl) || - String.IsNullOrEmpty(StorageAccountName) || String.IsNullOrEmpty(StorageAccountKey)) + if (String.IsNullOrEmpty(BatchAccountName) || + String.IsNullOrEmpty(BatchAccountKey) || + String.IsNullOrEmpty(BatchAccountUrl) || + String.IsNullOrEmpty(StorageAccountName) || + String.IsNullOrEmpty(StorageAccountKey)) { throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified."); } From 2ff8247083a75c0fd09f95a957da41a2c47bb655 Mon Sep 17 00:00:00 2001 From: Dan Lepow Date: Thu, 6 Sep 2018 12:47:55 -0700 Subject: [PATCH 4/9] format change --- BatchDotnetTutorialFfmpeg/Program.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/BatchDotnetTutorialFfmpeg/Program.cs b/BatchDotnetTutorialFfmpeg/Program.cs index 6f197af..901342a 100644 --- a/BatchDotnetTutorialFfmpeg/Program.cs +++ b/BatchDotnetTutorialFfmpeg/Program.cs @@ -45,10 +45,10 @@ public class Program public static void Main(string[] args) { - if (String.IsNullOrEmpty(BatchAccountName) || - String.IsNullOrEmpty(BatchAccountKey) || + if (String.IsNullOrEmpty(BatchAccountName) || + String.IsNullOrEmpty(BatchAccountKey) || String.IsNullOrEmpty(BatchAccountUrl) || - String.IsNullOrEmpty(StorageAccountName) || + String.IsNullOrEmpty(StorageAccountName) || String.IsNullOrEmpty(StorageAccountKey)) { throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified."); From 6d16e89a0cd08a8dc06bfe2e61d225b4403ddfac Mon Sep 17 00:00:00 2001 From: Dan Lepow Date: Fri, 7 Sep 2018 10:36:56 -0700 Subject: [PATCH 5/9] Incorp'd review comments --- .gitignore | 1 - BatchDotnetTutorialFfmpeg/Program.cs | 55 +++++++++++----------------- README.md | 2 +- 3 files changed, 23 insertions(+), 35 deletions(-) diff --git a/.gitignore b/.gitignore index 1cf8ba4..a52a718 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,6 @@ dlldata.c project.lock.json project.fragment.lock.json artifacts/ -# **/Properties/launchSettings.json *_i.c *_p.c diff --git a/BatchDotnetTutorialFfmpeg/Program.cs b/BatchDotnetTutorialFfmpeg/Program.cs index 901342a..8dcc8af 100644 --- a/BatchDotnetTutorialFfmpeg/Program.cs +++ b/BatchDotnetTutorialFfmpeg/Program.cs @@ -132,19 +132,19 @@ private static async Task MainAsync() using (BatchClient batchClient = BatchClient.Open(sharedKeyCredentials)) { // Create the Batch pool, which contains the compute nodes that execute the tasks. - CreatePoolIfNotExist(batchClient, PoolId); + await CreatePoolIfNotExistAsync(batchClient, PoolId); // Create the job that runs the tasks. - CreateJobIfNotExist(batchClient, JobId, PoolId); + await CreateJobAsync(batchClient, JobId, PoolId); // Create a collection of tasks and add them to the Batch job. // Provide a shared access signature for the tasks so that they can upload their output // to the Storage container. - AddTasks(batchClient, JobId, inputFiles, outputContainerSasUrl); + await AddTasksAsync(batchClient, JobId, inputFiles, outputContainerSasUrl); // Monitor task success or failure, specifying a maximum amount of time to wait for // the tasks to complete. - MonitorTasks(batchClient, JobId, TimeSpan.FromMinutes(30)); + await MonitorTasks(batchClient, JobId, TimeSpan.FromMinutes(30)); // Delete input container in storage Console.WriteLine("Deleting container [{0}]...", inputContainerName); @@ -163,14 +163,14 @@ private static async Task MainAsync() string response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { - batchClient.JobOperations.DeleteJob(JobId); + await batchClient.JobOperations.DeleteJobAsync(JobId); } Console.Write("Delete pool? [yes] no: "); response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { - batchClient.PoolOperations.DeletePool(PoolId); + await batchClient.PoolOperations.DeletePoolAsync(PoolId); } } } @@ -280,7 +280,7 @@ private static string GetContainerSasUrl(CloudBlobClient blobClient, string cont /// /// A BatchClient object /// ID of the CloudPool object to create. - private static void CreatePoolIfNotExist(BatchClient batchClient, string poolId) + private static async Task CreatePoolIfNotExistAsync(BatchClient batchClient, string poolId) { CloudPool pool = null; try @@ -322,7 +322,7 @@ private static void CreatePoolIfNotExist(BatchClient batchClient, string poolId) } }; - pool.Commit(); + await pool.CommitAsync(); } catch (BatchException be) { @@ -344,31 +344,18 @@ private static void CreatePoolIfNotExist(BatchClient batchClient, string poolId) /// A BatchClient object. /// ID of the job to create. /// ID of the CloudPool object in which to create the job. - private static void CreateJobIfNotExist(BatchClient batchClient, string jobId, string poolId) + private static async Task CreateJobAsync(BatchClient batchClient, string jobId, string poolId) { - try - { + Console.WriteLine("Creating job [{0}]...", jobId); CloudJob job = batchClient.JobOperations.CreateJob(); job.Id = jobId; job.PoolInformation = new PoolInformation { PoolId = poolId }; - job.Commit(); - } - catch (BatchException be) - { - // Accept the specific error code JobExists as that is expected if the job already exists - if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists) - { - Console.WriteLine("The job {0} already existed when we tried to create it", jobId); - } - else - { - throw; // Any other exception is unexpected - } - } + await job.CommitAsync(); } + /// /// @@ -381,7 +368,7 @@ private static void CreateJobIfNotExist(BatchClient batchClient, string jobId, s /// The shared access signature URL for the Azure /// Storagecontainer that will hold the output files that the tasks create. /// A collection of the submitted cloud tasks. - private static List AddTasks(BatchClient batchClient, string jobId, List inputFiles, string outputContainerSasUrl) + private static async Task> AddTasksAsync(BatchClient batchClient, string jobId, List inputFiles, string outputContainerSasUrl) { Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, jobId); @@ -422,7 +409,7 @@ private static List AddTasks(BatchClient batchClient, string jobId, L // Call BatchClient.JobOperations.AddTask() to add the tasks as a collection rather than making a // separate call for each. Bulk task submission helps to ensure efficient underlying API // calls to the Batch service. - batchClient.JobOperations.AddTask(jobId, tasks); + await batchClient.JobOperations.AddTaskAsync(jobId, tasks); return tasks; } @@ -433,7 +420,7 @@ private static List AddTasks(BatchClient batchClient, string jobId, L /// A BatchClient object. /// ID of the job containing the tasks to be monitored. /// The period of time to wait for the tasks to reach the completed state. - private static void MonitorTasks(BatchClient batchClient, string jobId, TimeSpan timeout) + private static async Task MonitorTasks(BatchClient batchClient, string jobId, TimeSpan timeout) { bool allTasksSuccessful = true; const string successMessage = "All tasks reached state Completed."; @@ -445,7 +432,7 @@ private static void MonitorTasks(BatchClient batchClient, string jobId, TimeSpan ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id"); - IEnumerable addedTasks = batchClient.JobOperations.ListTasks(jobId, detail); + List addedTasks = await batchClient.JobOperations.ListTasks(jobId, detail).ToListAsync(); Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout.ToString()); @@ -455,14 +442,15 @@ private static void MonitorTasks(BatchClient batchClient, string jobId, TimeSpan TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor(); try { - batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout); + await taskStateMonitor.WhenAll(addedTasks, TaskState.Completed, timeout); } catch (TimeoutException) { - batchClient.JobOperations.TerminateJob(jobId, failureMessage); + await batchClient.JobOperations.TerminateJobAsync(jobId, failureMessage); Console.WriteLine(failureMessage); + return false; } - batchClient.JobOperations.TerminateJob(jobId, successMessage); + await batchClient.JobOperations.TerminateJobAsync(jobId, successMessage); // All tasks have reached the "Completed" state, however, this does not guarantee all tasks completed successfully. // Here we further check each task's ExecutionInformation property to ensure that it did not encounter a scheduling error @@ -471,7 +459,7 @@ private static void MonitorTasks(BatchClient batchClient, string jobId, TimeSpan // Update the detail level to populate only the task id and executionInfo properties. detail.SelectClause = "id, executionInfo"; - IEnumerable completedTasks = batchClient.JobOperations.ListTasks(jobId, detail); + List completedTasks = await batchClient.JobOperations.ListTasks(jobId, detail).ToListAsync(); foreach (CloudTask task in completedTasks) { @@ -498,6 +486,7 @@ private static void MonitorTasks(BatchClient batchClient, string jobId, TimeSpan { Console.WriteLine("Success! All tasks completed successfully within the specified timeout period. Output files uploaded to output container."); } + return allTasksSuccessful; } } } diff --git a/README.md b/README.md index ebee37a..2ccad21 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ For details and explanation, see the accompanying article [Run a parallel worklo ## Prerequisites - Azure Batch account and linked general-purpose Azure Storage account -- Visual Studio 2017, or [.NET core 2.1](https://www.microsoft.com/net/download/dotnet-core/2.1) for Linux, macOS, or Windows +- Visual Studio 2017, or [.NET Core 2.1](https://www.microsoft.com/net/download/dotnet-core/2.1) for Linux, macOS, or Windows - Add ffmpeg as an [application package](https://docs.microsoft.com/azure/batch/batch-application-packages) to your Batch account (Application Id: *ffmpeg*, Version: *3.4*) ## Resources From ca6eeffbd0700343fe32060221d3372f9ed6bfd5 Mon Sep 17 00:00:00 2001 From: Dan Lepow Date: Fri, 7 Sep 2018 12:01:18 -0700 Subject: [PATCH 6/9] restored storageConnectionString definition --- BatchDotnetTutorialFfmpeg/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/BatchDotnetTutorialFfmpeg/Program.cs b/BatchDotnetTutorialFfmpeg/Program.cs index 8dcc8af..d97648f 100644 --- a/BatchDotnetTutorialFfmpeg/Program.cs +++ b/BatchDotnetTutorialFfmpeg/Program.cs @@ -86,8 +86,8 @@ private static async Task MainAsync() timer.Start(); // Construct the Storage account connection string - string storageConnectionString = - $"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey}"; + string storageConnectionString = String.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", + StorageAccountName, StorageAccountKey); // Retrieve the storage account CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); From cb24b4af2a6951d23ba06b2cb20959a5c77ffa31 Mon Sep 17 00:00:00 2001 From: Dan Lepow Date: Fri, 7 Sep 2018 12:38:39 -0700 Subject: [PATCH 7/9] Restored ffmpeg dowload link to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2ccad21..89dbf77 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ For details and explanation, see the accompanying article [Run a parallel worklo - Azure Batch account and linked general-purpose Azure Storage account - Visual Studio 2017, or [.NET Core 2.1](https://www.microsoft.com/net/download/dotnet-core/2.1) for Linux, macOS, or Windows +- Windows 64-bit version of [ffmpeg 3.4](https://ffmpeg.zeranoe.com/builds/win64/static/ffmpeg-3.4-win64-static.zip) - Add ffmpeg as an [application package](https://docs.microsoft.com/azure/batch/batch-application-packages) to your Batch account (Application Id: *ffmpeg*, Version: *3.4*) ## Resources From 0a54393cb50e3a2390b06f8838460a3588c6688e Mon Sep 17 00:00:00 2001 From: Dan Lepow Date: Mon, 10 Sep 2018 10:12:44 -0700 Subject: [PATCH 8/9] Updated monitoring procedure --- BatchDotnetTutorialFfmpeg/Program.cs | 52 +++++++++++----------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/BatchDotnetTutorialFfmpeg/Program.cs b/BatchDotnetTutorialFfmpeg/Program.cs index d97648f..c4f0f1a 100644 --- a/BatchDotnetTutorialFfmpeg/Program.cs +++ b/BatchDotnetTutorialFfmpeg/Program.cs @@ -7,6 +7,7 @@ namespace BatchDotnetTutorialFfmpeg using System.Collections.Generic; using System.Diagnostics; using System.IO; + using System.Linq; using System.Threading.Tasks; using Microsoft.Azure.Batch; using Microsoft.Azure.Batch.Auth; @@ -423,8 +424,10 @@ private static async Task> AddTasksAsync(BatchClient batchClient private static async Task MonitorTasks(BatchClient batchClient, string jobId, TimeSpan timeout) { bool allTasksSuccessful = true; - const string successMessage = "All tasks reached state Completed."; - const string failureMessage = "One or more tasks failed to reach the Completed state within the timeout period."; + const string completeMessage = "All tasks reached state Completed."; + const string incompleteMessage = "One or more tasks failed to reach the Completed state within the timeout period."; + const string successMessage = "Success! All tasks completed successfully. Output files uploaded to output container."; + const string failureMessage = "One or more tasks failed."; // Obtain the collection of tasks currently managed by the job. // Use a detail level to specify that only the "id" property of each task should be populated. @@ -446,47 +449,34 @@ private static async Task MonitorTasks(BatchClient batchClient, string job } catch (TimeoutException) { - await batchClient.JobOperations.TerminateJobAsync(jobId, failureMessage); - Console.WriteLine(failureMessage); + await batchClient.JobOperations.TerminateJobAsync(jobId); + Console.WriteLine(incompleteMessage); return false; } - await batchClient.JobOperations.TerminateJobAsync(jobId, successMessage); + await batchClient.JobOperations.TerminateJobAsync(jobId); + Console.WriteLine(completeMessage); // All tasks have reached the "Completed" state, however, this does not guarantee all tasks completed successfully. - // Here we further check each task's ExecutionInformation property to ensure that it did not encounter a scheduling error - // or return a non-zero exit code. - - // Update the detail level to populate only the task id and executionInfo properties. - detail.SelectClause = "id, executionInfo"; + // Here we further check for any tasks with an execution result of "Failure". - List completedTasks = await batchClient.JobOperations.ListTasks(jobId, detail).ToListAsync(); + // Update the detail level to populate only the executionInfo property. + detail.SelectClause = "executionInfo"; + // Filter for tasks with 'Failure' result. + detail.FilterClause = "executionInfo/result eq 'Failure'"; - foreach (CloudTask task in completedTasks) + List failedTasks = await batchClient.JobOperations.ListTasks(jobId, detail).ToListAsync(); + + if (failedTasks.Any()) { - if (task.ExecutionInformation.Result == TaskExecutionResult.Failure) - { - // A task with failure information set indicates there was a problem with the task. It is important to note that - // the task's state can be "Completed," yet still have encountered a failure. - - allTasksSuccessful = false; - - Console.WriteLine("WARNING: Task [{0}] encountered a failure: {1}", task.Id, task.ExecutionInformation.FailureInformation.Message); - if (task.ExecutionInformation.ExitCode != 0) - { - // A non-zero exit code may indicate that the application executed by the task encountered an error - // during execution. As not every application returns non-zero on failure by default (e.g. robocopy), - // your implementation of error checking may differ from this example. - - Console.WriteLine("WARNING: Task [{0}] returned a non-zero exit code - this may indicate task execution or completion failure.", task.Id); - } - } + allTasksSuccessful = false; + Console.WriteLine(failureMessage); } if (allTasksSuccessful) { - Console.WriteLine("Success! All tasks completed successfully within the specified timeout period. Output files uploaded to output container."); + Console.WriteLine(successMessage); } - return allTasksSuccessful; + return allTasksSuccessful; } } } From 5e1dfe1805d1b58ec155647a87d6deb80cc10385 Mon Sep 17 00:00:00 2001 From: Dan Lepow Date: Mon, 10 Sep 2018 11:33:03 -0700 Subject: [PATCH 9/9] tweak --- BatchDotnetTutorialFfmpeg/Program.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/BatchDotnetTutorialFfmpeg/Program.cs b/BatchDotnetTutorialFfmpeg/Program.cs index c4f0f1a..a670b5f 100644 --- a/BatchDotnetTutorialFfmpeg/Program.cs +++ b/BatchDotnetTutorialFfmpeg/Program.cs @@ -471,8 +471,7 @@ private static async Task MonitorTasks(BatchClient batchClient, string job allTasksSuccessful = false; Console.WriteLine(failureMessage); } - - if (allTasksSuccessful) + else { Console.WriteLine(successMessage); }