diff --git a/.gitignore b/.gitignore index 940794e..a52a718 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,6 @@ dlldata.c project.lock.json project.fragment.lock.json artifacts/ -**/Properties/launchSettings.json *_i.c *_p.c diff --git a/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.csproj b/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.csproj index 4fec3d3..e613d28 100644 --- a/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.csproj +++ b/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.csproj @@ -2,13 +2,13 @@ Exe - net462 + netcoreapp2.1 Microsoft.Azure.Batch.Samples.BatchDotNetTutorialFfmpeg - - + + \ No newline at end of file diff --git a/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.sln b/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.sln index 3c1bdb5..98184b4 100644 --- a/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.sln +++ b/BatchDotnetTutorialFfmpeg/BatchDotnetTutorialFfmpeg.sln @@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 VisualStudioVersion = 15.0.26228.0 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BatchDotnetTutorialFfmpeg", "BatchDotnetTutorialFfmpeg.csproj", "{C8DFEFA7-5BC3-4217-88DC-791020D6A909}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BatchDotnetTutorialFfmpeg", "BatchDotnetTutorialFfmpeg.csproj", "{C8DFEFA7-5BC3-4217-88DC-791020D6A909}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{5CAC1578-E7D4-416E-BB9D-1A9800B0F1A6}" EndProject @@ -23,4 +23,7 @@ Global GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {40550F11-FA23-4798-B91D-D045F41F982C} + EndGlobalSection EndGlobal diff --git a/BatchDotnetTutorialFfmpeg/Program.cs b/BatchDotnetTutorialFfmpeg/Program.cs index f2db2d3..a670b5f 100644 --- a/BatchDotnetTutorialFfmpeg/Program.cs +++ b/BatchDotnetTutorialFfmpeg/Program.cs @@ -7,6 +7,8 @@ namespace BatchDotnetTutorialFfmpeg using System.Collections.Generic; using System.Diagnostics; using System.IO; + using System.Linq; + using System.Threading.Tasks; using Microsoft.Azure.Batch; using Microsoft.Azure.Batch.Auth; using Microsoft.Azure.Batch.Common; @@ -55,108 +57,125 @@ public static void Main(string[] args) try { - // START TIMER - Console.WriteLine("Sample start: {0}", DateTime.Now); + // Call the asynchronous version of the Main() method. This is done so that we can await various + // calls to async methods within the "Main" method of this console application. + MainAsync().Wait(); + } + catch (AggregateException) + { + Console.WriteLine(); + Console.WriteLine("One or more exceptions occurred."); Console.WriteLine(); - Stopwatch timer = new Stopwatch(); - timer.Start(); + } + finally + { + Console.WriteLine(); + Console.WriteLine("Sample complete, hit ENTER to exit..."); + Console.ReadLine(); + } + } + + /// + /// Provides an asynchronous version of the Main method, allowing for the awaiting of async method calls within. + /// + /// A object that represents the asynchronous operation. + private static async Task MainAsync() + { + Console.WriteLine("Sample start: {0}", DateTime.Now); + Console.WriteLine(); + Stopwatch timer = new Stopwatch(); + timer.Start(); + + // Construct the Storage account connection string + string storageConnectionString = String.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", + StorageAccountName, StorageAccountKey); + + // Retrieve the storage account + CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); - // STORAGE SETUP - // Construct the Storage account connection string - string storageConnectionString = String.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", - StorageAccountName, StorageAccountKey); + // Create the blob client, for use in obtaining references to blob storage containers + CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); - // Retrieve the storage account - CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); + + // Use the blob client to create the containers in blob storage + const string inputContainerName = "input"; + const string outputContainerName = "output"; - // Create the blob client, which will be used to obtain references to blob storage containers - CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); + await CreateContainerIfNotExistAsync(blobClient, inputContainerName); + await CreateContainerIfNotExistAsync(blobClient, outputContainerName); - // Use the blob client to create the containers in blob storage - const string inputContainerName = "input"; - const string outputContainerName = "output"; + // RESOURCE FILE SETUP + // Input files: Specify the location of the data files that the tasks process, and + // put them in a List collection. Make sure you have copied the data files to: + // \\InputFiles. - CreateContainerIfNotExist(blobClient, inputContainerName); - CreateContainerIfNotExist(blobClient, outputContainerName); - - // RESOURCE FILE SETUP - // Input files: Specify the location of the data files that the tasks process, and - // put them in a List collection. Make sure you have copied the data files to: - // \\InputFiles. + string inputPath = Path.Combine(Environment.CurrentDirectory, "InputFiles"); - List inputFilePaths = new List(Directory.GetFileSystemEntries(@"..\..\..\InputFiles", "*.mp4", + List inputFilePaths = new List(Directory.GetFileSystemEntries(inputPath, "*.mp4", SearchOption.TopDirectoryOnly)); - // Upload data files. - // Upload the data files using UploadResourceFilesToContainer(). This data will be - // processed by each of the tasks that are executed on the compute nodes within the pool. - List inputFiles = UploadResourceFilesToContainer(blobClient, inputContainerName, inputFilePaths); + // Upload data files. + // Upload the data files using UploadResourceFilesToContainer(). This data will be + // processed by each of the tasks that are executed on the compute nodes within the pool. + List inputFiles = await UploadFilesToContainerAsync(blobClient, inputContainerName, inputFilePaths); - // Obtain a shared access signature that provides write access to the output container to which - // the tasks will upload their output. - string outputContainerSasUrl = GetContainerSasUrl(blobClient, outputContainerName, SharedAccessBlobPermissions.Write); + // Obtain a shared access signature that provides write access to the output container to which + // the tasks will upload their output. + string outputContainerSasUrl = GetContainerSasUrl(blobClient, outputContainerName, SharedAccessBlobPermissions.Write); - // CREATE BATCH CLIENT / CREATE POOL / CREATE JOB / ADD TASKS + // CREATE BATCH CLIENT / CREATE POOL / CREATE JOB / ADD TASKS - // Create a Batch client and authenticate with shared key credentials. - // The Batch client allows the app to interact with the Batch service. - BatchSharedKeyCredentials sharedKeyCredentials = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey); + // Create a Batch client and authenticate with shared key credentials. + // The Batch client allows the app to interact with the Batch service. + BatchSharedKeyCredentials sharedKeyCredentials = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey); - using (BatchClient batchClient = BatchClient.Open(sharedKeyCredentials)) - { - // Create the Batch pool, which contains the compute nodes that execute the tasks. - CreatePoolIfNotExist(batchClient, PoolId); + using (BatchClient batchClient = BatchClient.Open(sharedKeyCredentials)) + { + // Create the Batch pool, which contains the compute nodes that execute the tasks. + await CreatePoolIfNotExistAsync(batchClient, PoolId); - // Create the job that runs the tasks. - CreateJobIfNotExist(batchClient, JobId, PoolId); + // Create the job that runs the tasks. + await CreateJobAsync(batchClient, JobId, PoolId); - // Create a collection of tasks and add them to the Batch job. - // Provide a shared access signature for the tasks so that they can upload their output - // to the Storage container. - AddTasks(batchClient, JobId, inputFiles, outputContainerSasUrl); + // Create a collection of tasks and add them to the Batch job. + // Provide a shared access signature for the tasks so that they can upload their output + // to the Storage container. + await AddTasksAsync(batchClient, JobId, inputFiles, outputContainerSasUrl); - // Monitor task success or failure, specifying a maximum amount of time to wait for - // the tasks to complete. - MonitorTasks(batchClient, JobId, TimeSpan.FromMinutes(30)); + // Monitor task success or failure, specifying a maximum amount of time to wait for + // the tasks to complete. + await MonitorTasks(batchClient, JobId, TimeSpan.FromMinutes(30)); - // Delete input container in storage - Console.WriteLine("Deleting container [{0}]...", inputContainerName); - CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName); - container.DeleteIfExists(); + // Delete input container in storage + Console.WriteLine("Deleting container [{0}]...", inputContainerName); + CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName); + await container.DeleteIfExistsAsync(); - // Print out timing info - timer.Stop(); - Console.WriteLine(); - Console.WriteLine("Sample end: {0}", DateTime.Now); - Console.WriteLine("Elapsed time: {0}", timer.Elapsed); - - // Clean up Batch resources (if the user so chooses) - Console.WriteLine(); - Console.Write("Delete job? [yes] no: "); - string response = Console.ReadLine().ToLower(); - if (response != "n" && response != "no") - { - batchClient.JobOperations.DeleteJob(JobId); - } + // Print out timing info + timer.Stop(); + Console.WriteLine(); + Console.WriteLine("Sample end: {0}", DateTime.Now); + Console.WriteLine("Elapsed time: {0}", timer.Elapsed); - Console.Write("Delete pool? [yes] no: "); - response = Console.ReadLine().ToLower(); - if (response != "n" && response != "no") - { - batchClient.PoolOperations.DeletePool(PoolId); - } - } - } - finally - { + // Clean up Batch resources (if the user so chooses) Console.WriteLine(); - Console.WriteLine("Sample complete, hit ENTER to exit..."); - Console.ReadLine(); + Console.Write("Delete job? [yes] no: "); + string response = Console.ReadLine().ToLower(); + if (response != "n" && response != "no") + { + await batchClient.JobOperations.DeleteJobAsync(JobId); + } + + Console.Write("Delete pool? [yes] no: "); + response = Console.ReadLine().ToLower(); + if (response != "n" && response != "no") + { + await batchClient.PoolOperations.DeletePoolAsync(PoolId); + } } } - - + // FUNCTION IMPLEMENTATIONS /// @@ -164,19 +183,12 @@ public static void Main(string[] args) /// /// A . /// The name for the new container. - - private static void CreateContainerIfNotExist(CloudBlobClient blobClient, string containerName) + + private static async Task CreateContainerIfNotExistAsync(CloudBlobClient blobClient, string containerName) { CloudBlobContainer container = blobClient.GetContainerReference(containerName); - - if (container.CreateIfNotExists()) - { - Console.WriteLine("Container [{0}] created.", containerName); - } - else - { - Console.WriteLine("Container [{0}] exists, skipping creation.", containerName); - } + await container.CreateIfNotExistsAsync(); + Console.WriteLine("Creating container [{0}].", containerName); } @@ -189,13 +201,13 @@ private static void CreateContainerIfNotExist(CloudBlobClient blobClient, string /// Name of the blob storage container to which the files are uploaded. /// A collection of paths of the files to be uploaded to the container. /// A collection of objects. - private static List UploadResourceFilesToContainer(CloudBlobClient blobClient, string containerName, List filePaths) + private static async Task> UploadFilesToContainerAsync(CloudBlobClient blobClient, string inputContainerName, List filePaths) { List resourceFiles = new List(); foreach (string filePath in filePaths) { - resourceFiles.Add(UploadResourceFileToContainer(blobClient, containerName, filePath)); + resourceFiles.Add(await UploadResourceFileToContainerAsync(blobClient, inputContainerName, filePath)); } return resourceFiles; @@ -208,7 +220,7 @@ private static List UploadResourceFilesToContainer(CloudBlobClient /// The name of the blob storage container to which the file should be uploaded. /// The full path to the file to upload to Storage. /// A ResourceFile object representing the file in blob storage. - private static ResourceFile UploadResourceFileToContainer(CloudBlobClient blobClient, string containerName, string filePath) + private static async Task UploadResourceFileToContainerAsync(CloudBlobClient blobClient, string containerName, string filePath) { Console.WriteLine("Uploading file {0} to container [{1}]...", filePath, containerName); @@ -217,7 +229,7 @@ private static ResourceFile UploadResourceFileToContainer(CloudBlobClient blobCl CloudBlobContainer container = blobClient.GetContainerReference(containerName); CloudBlockBlob blobData = container.GetBlockBlobReference(blobName); - blobData.UploadFromFile(filePath); + await blobData.UploadFromFileAsync(filePath); // Set the expiry time and permissions for the blob shared access signature. In this case, no start time is specified, // so the shared access signature becomes valid immediately @@ -269,7 +281,7 @@ private static string GetContainerSasUrl(CloudBlobClient blobClient, string cont /// /// A BatchClient object /// ID of the CloudPool object to create. - private static void CreatePoolIfNotExist(BatchClient batchClient, string poolId) + private static async Task CreatePoolIfNotExistAsync(BatchClient batchClient, string poolId) { CloudPool pool = null; try @@ -311,7 +323,7 @@ private static void CreatePoolIfNotExist(BatchClient batchClient, string poolId) } }; - pool.Commit(); + await pool.CommitAsync(); } catch (BatchException be) { @@ -333,31 +345,18 @@ private static void CreatePoolIfNotExist(BatchClient batchClient, string poolId) /// A BatchClient object. /// ID of the job to create. /// ID of the CloudPool object in which to create the job. - private static void CreateJobIfNotExist(BatchClient batchClient, string jobId, string poolId) + private static async Task CreateJobAsync(BatchClient batchClient, string jobId, string poolId) { - try - { + Console.WriteLine("Creating job [{0}]...", jobId); CloudJob job = batchClient.JobOperations.CreateJob(); job.Id = jobId; job.PoolInformation = new PoolInformation { PoolId = poolId }; - job.Commit(); - } - catch (BatchException be) - { - // Accept the specific error code JobExists as that is expected if the job already exists - if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists) - { - Console.WriteLine("The job {0} already existed when we tried to create it", jobId); - } - else - { - throw; // Any other exception is unexpected - } - } + await job.CommitAsync(); } + /// /// @@ -370,7 +369,7 @@ private static void CreateJobIfNotExist(BatchClient batchClient, string jobId, s /// The shared access signature URL for the Azure /// Storagecontainer that will hold the output files that the tasks create. /// A collection of the submitted cloud tasks. - private static List AddTasks(BatchClient batchClient, string jobId, List inputFiles, string outputContainerSasUrl) + private static async Task> AddTasksAsync(BatchClient batchClient, string jobId, List inputFiles, string outputContainerSasUrl) { Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, jobId); @@ -411,7 +410,7 @@ private static List AddTasks(BatchClient batchClient, string jobId, L // Call BatchClient.JobOperations.AddTask() to add the tasks as a collection rather than making a // separate call for each. Bulk task submission helps to ensure efficient underlying API // calls to the Batch service. - batchClient.JobOperations.AddTask(jobId, tasks); + await batchClient.JobOperations.AddTaskAsync(jobId, tasks); return tasks; } @@ -422,11 +421,13 @@ private static List AddTasks(BatchClient batchClient, string jobId, L /// A BatchClient object. /// ID of the job containing the tasks to be monitored. /// The period of time to wait for the tasks to reach the completed state. - private static void MonitorTasks(BatchClient batchClient, string jobId, TimeSpan timeout) + private static async Task MonitorTasks(BatchClient batchClient, string jobId, TimeSpan timeout) { bool allTasksSuccessful = true; - const string successMessage = "All tasks reached state Completed."; - const string failureMessage = "One or more tasks failed to reach the Completed state within the timeout period."; + const string completeMessage = "All tasks reached state Completed."; + const string incompleteMessage = "One or more tasks failed to reach the Completed state within the timeout period."; + const string successMessage = "Success! All tasks completed successfully. Output files uploaded to output container."; + const string failureMessage = "One or more tasks failed."; // Obtain the collection of tasks currently managed by the job. // Use a detail level to specify that only the "id" property of each task should be populated. @@ -434,7 +435,7 @@ private static void MonitorTasks(BatchClient batchClient, string jobId, TimeSpan ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id"); - IEnumerable addedTasks = batchClient.JobOperations.ListTasks(jobId, detail); + List addedTasks = await batchClient.JobOperations.ListTasks(jobId, detail).ToListAsync(); Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout.ToString()); @@ -444,49 +445,37 @@ private static void MonitorTasks(BatchClient batchClient, string jobId, TimeSpan TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor(); try { - batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout); + await taskStateMonitor.WhenAll(addedTasks, TaskState.Completed, timeout); } catch (TimeoutException) { - batchClient.JobOperations.TerminateJob(jobId, failureMessage); - Console.WriteLine(failureMessage); + await batchClient.JobOperations.TerminateJobAsync(jobId); + Console.WriteLine(incompleteMessage); + return false; } - batchClient.JobOperations.TerminateJob(jobId, successMessage); + await batchClient.JobOperations.TerminateJobAsync(jobId); + Console.WriteLine(completeMessage); // All tasks have reached the "Completed" state, however, this does not guarantee all tasks completed successfully. - // Here we further check each task's ExecutionInformation property to ensure that it did not encounter a scheduling error - // or return a non-zero exit code. - - // Update the detail level to populate only the task id and executionInfo properties. - detail.SelectClause = "id, executionInfo"; + // Here we further check for any tasks with an execution result of "Failure". - IEnumerable completedTasks = batchClient.JobOperations.ListTasks(jobId, detail); + // Update the detail level to populate only the executionInfo property. + detail.SelectClause = "executionInfo"; + // Filter for tasks with 'Failure' result. + detail.FilterClause = "executionInfo/result eq 'Failure'"; - foreach (CloudTask task in completedTasks) + List failedTasks = await batchClient.JobOperations.ListTasks(jobId, detail).ToListAsync(); + + if (failedTasks.Any()) { - if (task.ExecutionInformation.Result == TaskExecutionResult.Failure) - { - // A task with failure information set indicates there was a problem with the task. It is important to note that - // the task's state can be "Completed," yet still have encountered a failure. - - allTasksSuccessful = false; - - Console.WriteLine("WARNING: Task [{0}] encountered a failure: {1}", task.Id, task.ExecutionInformation.FailureInformation.Message); - if (task.ExecutionInformation.ExitCode != 0) - { - // A non-zero exit code may indicate that the application executed by the task encountered an error - // during execution. As not every application returns non-zero on failure by default (e.g. robocopy), - // your implementation of error checking may differ from this example. - - Console.WriteLine("WARNING: Task [{0}] returned a non-zero exit code - this may indicate task execution or completion failure.", task.Id); - } - } + allTasksSuccessful = false; + Console.WriteLine(failureMessage); } - - if (allTasksSuccessful) + else { - Console.WriteLine("Success! All tasks completed successfully within the specified timeout period. Output files uploaded to output container."); + Console.WriteLine(successMessage); } + return allTasksSuccessful; } } } diff --git a/BatchDotnetTutorialFfmpeg/Properties/launchSettings.json b/BatchDotnetTutorialFfmpeg/Properties/launchSettings.json new file mode 100644 index 0000000..4e92bea --- /dev/null +++ b/BatchDotnetTutorialFfmpeg/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "BatchDotnetTutorialFfmpeg": { + "commandName": "Project", + "workingDirectory": "$(ProjectDir)" + } + } +} \ No newline at end of file diff --git a/README.md b/README.md index 76a34ca..89dbf77 100644 --- a/README.md +++ b/README.md @@ -14,14 +14,14 @@ For details and explanation, see the accompanying article [Run a parallel worklo ## Prerequisites - Azure Batch account and linked general-purpose Azure Storage account -- Visual Studio 2017 +- Visual Studio 2017, or [.NET Core 2.1](https://www.microsoft.com/net/download/dotnet-core/2.1) for Linux, macOS, or Windows - Windows 64-bit version of [ffmpeg 3.4](https://ffmpeg.zeranoe.com/builds/win64/static/ffmpeg-3.4-win64-static.zip) - Add ffmpeg as an [application package](https://docs.microsoft.com/azure/batch/batch-application-packages) to your Batch account (Application Id: *ffmpeg*, Version: *3.4*) ## Resources - [Azure Batch documentation](https://docs.microsoft.com/azure/batch/) -- [Azure Batch code samples repo](https://github.com/Azure/azure-batch-samples) +- [Azure Batch code samples repo](https://github.com/Azure-Samples/azure-batch-samples) ## Project code of conduct