Improve stalled and failed imports

.github/ISSUE_TEMPLATE/1-bug.yml

@@ -1,6 +1,6 @@
 name: Bug report
 description: File a bug report if something is not working right.
-title: "[BUG]: "
+title: "[BUG] "
 labels: ["bug"]
 body:
   - type: markdown
@@ -40,6 +40,7 @@ body:
         - Windows
         - Linux
         - MacOS
+        - Unraid
     validations:
       required: true
   - type: dropdown

.github/ISSUE_TEMPLATE/2-feature.yml

@@ -1,6 +1,6 @@
 name: Feature request
 description: File a feature request.
-title: "[FEATURE]: "
+title: "[FEATURE] "
 labels: ["enhancement"]
 body:
   - type: markdown

.github/ISSUE_TEMPLATE/3-help.yml

@@ -1,6 +1,6 @@
 name: Help request
 description: Ask a question to receive help.
-title: "[HELP]: "
+title: "[HELP] "
 labels: ["question"]
 body:
   - type: markdown

README.md

@@ -73,9 +73,10 @@ This tool is actively developed and still a work in progress. Join the Discord s
 
 1. Set `QUEUECLEANER__ENABLED` to `true`.
 2. Set `QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES` to a desired value.
-3. Set `DOWNLOAD_CLIENT` to `none`.
+3. Optionally set failed import message patterns to ignore using `QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__<NUMBER>`.
+4. Set `DOWNLOAD_CLIENT` to `none`.
 
-**No other action involving a download client would work (e.g. content blocking, removing stalled downloads etc.).**
+**No other action involving a download client would work (e.g. content blocking, removing stalled downloads, excluding private trackers).**
 
 ## Usage
 
@@ -99,7 +100,11 @@ services:
       - QUEUECLEANER__ENABLED=true
       - QUEUECLEANER__RUNSEQUENTIALLY=true
       - QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5
+      - QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=false
+      # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=title mismatch
+      # - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1=manual import required
       - QUEUECLEANER__STALLED_MAX_STRIKES=5
+      - QUEUECLEANER__STALLED_IGNORE_PRIVATE=false
 
       - CONTENTBLOCKER__ENABLED=true
       - CONTENTBLOCKER__BLACKLIST__ENABLED=true
@@ -155,7 +160,10 @@ services:
 | QUEUECLEANER__ENABLED | No | Enable or disable the queue cleaner | true |
 | QUEUECLEANER__RUNSEQUENTIALLY | No | If set to true, the queue cleaner will run after the content blocker instead of running in parallel, streamlining the cleaning process | true |
 | QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES | No | After how many strikes should a failed import be removed<br>0 means never | 0 |
+| QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE | No | Whether to ignore failed imports from private trackers | false |
+| QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0 | No | First pattern to look for when an import is failed<br>If the specified message pattern is found, the item is skipped | empty |
 | QUEUECLEANER__STALLED_MAX_STRIKES | No | After how many strikes should a stalled download be removed<br>0 means never | 0 |
+| QUEUECLEANER__STALLED_IGNORE_PRIVATE | No | Whether to ignore stalled downloads from private trackers | false |
 |||||
 | CONTENTBLOCKER__ENABLED | No | Enable or disable the content blocker | false |
 | CONTENTBLOCKER__BLACKLIST__ENABLED | Yes if content blocker is enabled and whitelist is not enabled | Enable or disable the blacklist | false |
@@ -203,6 +211,10 @@ regex:<ANY_REGEX>   // regex that needs to be marked at the start of the line wi
 SONARR__INSTANCES__<NUMBER>__URL
 SONARR__INSTANCES__<NUMBER>__APIKEY
 ```
+6. Multiple failed import patterns can be specified using this format, where `<NUMBER>` starts from 0:
+```
+QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__<NUMBER>
+```
 
 #
 

code/Common/Configuration/QueueCleaner/QueueCleanerConfig.cs

@@ -13,8 +13,17 @@ public sealed record QueueCleanerConfig : IJobConfig
     [ConfigurationKeyName("IMPORT_FAILED_MAX_STRIKES")]
     public ushort ImportFailedMaxStrikes { get; init; }
 
+    [ConfigurationKeyName("IMPORT_FAILED_IGNORE_PRIVATE")]
+    public bool ImportFailedIgnorePrivate { get; init; }
+
+    [ConfigurationKeyName("IMPORT_FAILED_IGNORE_PATTERNS")]
+    public List<string>? ImportFailedIgnorePatterns { get; init; }
+
     [ConfigurationKeyName("STALLED_MAX_STRIKES")]
     public ushort StalledMaxStrikes { get; init; }
+
+    [ConfigurationKeyName("STALLED_IGNORE_PRIVATE")]
+    public bool StalledIgnorePrivate { get; init; }
 
     public void Validate()
     {

code/Domain/Models/Arr/Queue/QueueRecord.cs

@@ -1,15 +1,16 @@
-namespace Domain.Models.Arr.Queue;
+namespace Domain.Models.Arr.Queue;
 
-public record QueueRecord
+public sealed record QueueRecord
 {
     public int SeriesId { get; init; }
     public int EpisodeId { get; init; }
     public int SeasonNumber { get; init; }
     public int MovieId { get; init; }
     public required string Title { get; init; }
     public string Status { get; init; }
     public string TrackedDownloadStatus { get; init; }
     public string TrackedDownloadState { get; init; }
+    public List<TrackedDownloadStatusMessage>? StatusMessages { get; init; }
     public required string DownloadId { get; init; }
     public required string Protocol { get; init; }
     public required int Id { get; init; }

code/Domain/Models/Arr/Queue/TrackedDownloadStatusMessage.cs

@@ -0,0 +1,8 @@
+namespace Domain.Models.Arr.Queue;
+
+public sealed record TrackedDownloadStatusMessage
+{
+    public string Title { get; set; }
+
+    public List<string>? Messages { get; set; }
+}

code/Domain/Models/Deluge/Response/TorrentStatus.cs

@@ -2,11 +2,13 @@
 
 public sealed record TorrentStatus
 {
-    public string? Hash { get; set; }
+    public string? Hash { get; init; }
 
-    public string? State { get; set; }
+    public string? State { get; init; }
 
-    public string? Name { get; set; }
+    public string? Name { get; init; }
 
-    public ulong Eta { get; set; }
+    public ulong Eta { get; init; }
+
+    public bool Private { get; init; }
 }

code/Executable/appsettings.Development.json

@@ -26,7 +26,12 @@
     "Enabled": true,
     "RunSequentially": true,
     "IMPORT_FAILED_MAX_STRIKES": 5,
-    "STALLED_MAX_STRIKES": 5
+    "IMPORT_FAILED_IGNORE_PRIVATE": true,
+    "IMPORT_FAILED_IGNORE_PATTERNS": [
+      "file is a sample"
+    ],
+    "STALLED_MAX_STRIKES": 5,
+    "STALLED_IGNORE_PRIVATE": true
   },
   "DOWNLOAD_CLIENT": "qbittorrent",
   "qBittorrent": {

code/Executable/appsettings.json

@@ -26,7 +26,10 @@
     "Enabled": true,
     "RunSequentially": true,
     "IMPORT_FAILED_MAX_STRIKES": 5,
-    "STALLED_MAX_STRIKES": 5
+    "IMPORT_FAILED_IGNORE_PRIVATE": false,
+    "IMPORT_FAILED_IGNORE_PATTERNS": [],
+    "STALLED_MAX_STRIKES": 5,
+    "STALLED_IGNORE_PRIVATE": false
   },
   "DOWNLOAD_CLIENT": "qbittorrent",
   "qBittorrent": {

code/Infrastructure/Verticals/Arr/ArrClient.cs

@@ -65,17 +65,30 @@ public virtual async Task<QueueListResponse> GetQueueItemsAsync(ArrInstance arrI
         return queueResponse;
     }
 
-    public virtual bool ShouldRemoveFromQueue(QueueRecord record)
+    public virtual bool ShouldRemoveFromQueue(QueueRecord record, bool isPrivateDownload)
     {
+        if (_queueCleanerConfig.ImportFailedIgnorePrivate && isPrivateDownload)
+        {
+            // ignore private trackers
+            _logger.LogDebug("skip failed import check | download is private | {name}", record.Title);
+            return false;
+        }
+
         bool hasWarn() => record.TrackedDownloadStatus
             .Equals("warning", StringComparison.InvariantCultureIgnoreCase);
         bool isImportBlocked() => record.TrackedDownloadState
             .Equals("importBlocked", StringComparison.InvariantCultureIgnoreCase);
         bool isImportPending() => record.TrackedDownloadState
             .Equals("importPending", StringComparison.InvariantCultureIgnoreCase);
-
+        
         if (hasWarn() && (isImportBlocked() || isImportPending()))
         {
+            if (HasIgnoredPatterns(record))
+            {
+                _logger.LogDebug("skip failed import check | contains ignored pattern | {name}", record.Title);
+                return false;
+            }
+
             return _striker.StrikeAndCheckLimit(
                 record.DownloadId,
                 record.Title,
@@ -134,4 +147,32 @@ protected virtual void SetApiKey(HttpRequestMessage request, string apiKey)
     {
         request.Headers.Add("x-api-key", apiKey);
     }
+
+    private bool HasIgnoredPatterns(QueueRecord record)
+    {
+        if (_queueCleanerConfig.ImportFailedIgnorePatterns?.Count is null or 0)
+        {
+            // no patterns are configured
+            return false;
+        }
+
+        if (record.StatusMessages?.Count is null or 0)
+        {
+            // no status message found
+            return false;
+        }
+
+        HashSet<string> messages = record.StatusMessages
+            .SelectMany(x => x.Messages ?? Enumerable.Empty<string>())
+            .ToHashSet();
+        record.StatusMessages.Select(x => x.Title)
+            .ToList()
+            .ForEach(x => messages.Add(x));
+
+        return messages.Any(
+            m => _queueCleanerConfig.ImportFailedIgnorePatterns.Any(
+                p => !string.IsNullOrWhiteSpace(p.Trim()) && m.Contains(p, StringComparison.InvariantCultureIgnoreCase)
+            )
+        );
+    }
 }

code/Infrastructure/Verticals/DownloadClient/Deluge/DelugeService.cs

@@ -30,18 +30,19 @@ public override async Task LoginAsync()
         await _client.LoginAsync();
     }
 
-    public override async Task<bool> ShouldRemoveFromArrQueueAsync(string hash)
+    public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
     {
         hash = hash.ToLowerInvariant();
 
         DelugeContents? contents = null;
+        RemoveResult result = new();
 
         TorrentStatus? status = await GetTorrentStatus(hash);
 
         if (status?.Hash is null)
         {
             _logger.LogDebug("failed to find torrent {hash} in the download client", hash);
-            return false;
+            return result;
         }
 
         try
@@ -63,7 +64,10 @@ public override async Task<bool> ShouldRemoveFromArrQueueAsync(string hash)
             }
         });
 
-        return shouldRemove || IsItemStuckAndShouldRemove(status);
+        result.ShouldRemove = shouldRemove || IsItemStuckAndShouldRemove(status);
+        result.IsPrivate = status.Private;
+
+        return result;
     }
 
     public override async Task BlockUnwantedFilesAsync(string hash)
@@ -128,6 +132,18 @@ public override async Task BlockUnwantedFilesAsync(string hash)
 
     private bool IsItemStuckAndShouldRemove(TorrentStatus status)
     {
+        if (_queueCleanerConfig.StalledMaxStrikes is 0)
+        {
+            return false;
+        }
+
+        if (_queueCleanerConfig.StalledIgnorePrivate && status.Private)
+        {
+            // ignore private trackers
+            _logger.LogDebug("skip stalled check | download is private | {name}", status.Name);
+            return false;
+        }
+
         if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase))
         {
             return false;
@@ -146,7 +162,7 @@ private bool IsItemStuckAndShouldRemove(TorrentStatus status)
         return await _client.SendRequest<TorrentStatus?>(
             "web.get_torrent_status",
             hash,
-            new[] { "hash", "state", "name", "eta" }
+            new[] { "hash", "state", "name", "eta", "private" }
         );
     }
 

code/Infrastructure/Verticals/DownloadClient/DownloadServiceBase.cs

@@ -31,7 +31,7 @@ Striker striker
 
     public abstract Task LoginAsync();
 
-    public abstract Task<bool> ShouldRemoveFromArrQueueAsync(string hash);
+    public abstract Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash);
 
     public abstract Task BlockUnwantedFilesAsync(string hash);
 

code/Infrastructure/Verticals/DownloadClient/DummyDownloadService.cs

@@ -21,7 +21,7 @@ public override Task LoginAsync()
         return Task.CompletedTask;
     }
 
-    public override Task<bool> ShouldRemoveFromArrQueueAsync(string hash)
+    public override Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
     {
         throw new NotImplementedException();
     }

code/Infrastructure/Verticals/DownloadClient/IDownloadService.cs

@@ -4,7 +4,7 @@ public interface IDownloadService : IDisposable
 {
     public Task LoginAsync();
 
-    public Task<bool> ShouldRemoveFromArrQueueAsync(string hash);
+    public Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash);
 
     public Task BlockUnwantedFilesAsync(string hash);
 }