Do not log cmk errors at critical level (#3272)

* do not log cmk errors at critical level

* wait for polling time on failure

* fix tests

* add the delay at the start of the loop instead of the end

src/Microsoft.Health.Dicom.Api.UnitTests/Features/BackgroundServices/DeletedInstanceCleanupWorkerTests.cs

@@ -135,7 +135,7 @@ public async Task GivenMetrics_WhenEmitting_ThenSendMetrics()
                 return new DeleteMetrics { OldestDeletion = oldest, TotalExhaustedRetries = TotalExhausted };
             });
 
-        await Assert.ThrowsAsync<TaskCanceledException>(() => _deletedInstanceCleanupWorker.ExecuteAsync(_cancellationTokenSource.Token));
+        await _deletedInstanceCleanupWorker.ExecuteAsync(_cancellationTokenSource.Token);
 
         // Force the meter provides to emit the metrics earlier than they might otherwise
         _meterProvider.ForceFlush();

src/Microsoft.Health.Dicom.Api/Features/BackgroundServices/DeletedInstanceCleanupWorker.cs

@@ -8,13 +8,15 @@
 using System.Threading;
 using System.Threading.Tasks;
 using EnsureThat;
+using Microsoft.Data.SqlClient;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Microsoft.Health.Dicom.Core.Configs;
 using Microsoft.Health.Dicom.Core.Exceptions;
 using Microsoft.Health.Dicom.Core.Features.Delete;
 using Microsoft.Health.Dicom.Core.Features.Telemetry;
 using Microsoft.Health.Dicom.Core.Models.Delete;
+using Microsoft.Health.SqlServer.Features.Storage;
 
 namespace Microsoft.Health.Dicom.Api.Features.BackgroundServices;
 
@@ -51,14 +53,14 @@ public async Task ExecuteAsync(CancellationToken stoppingToken)
         {
             try
             {
+                await Task.Delay(_pollingInterval, stoppingToken).ConfigureAwait(false);
+
                 // Send metrics related to deletion progress
                 DeleteMetrics metrics = await _deleteService.GetMetricsAsync(stoppingToken);
 
                 _deleteMeter.OldestRequestedDeletion.Add(metrics.OldestDeletion.ToUnixTimeSeconds());
                 _deleteMeter.CountDeletionsMaxRetry.Add(metrics.TotalExhaustedRetries);
 
-                await Task.Delay(_pollingInterval, stoppingToken);
-
                 // Delete all instances pending deletion
                 bool success;
                 int retrievedInstanceCount;
@@ -72,6 +74,10 @@ public async Task ExecuteAsync(CancellationToken stoppingToken)
             {
                 _logger.LogInformation("The data store is not currently ready. Processing will continue after the next wait period.");
             }
+            catch (SqlException sqlEx) when (sqlEx.IsCMKError())
+            {
+                _logger.LogInformation(sqlEx, "The customer-managed key is misconfigured by the customer.");
+            }
             catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
             {
                 // Cancel requested.