diff --git a/source/Playnite/Database/GameDatabase.cs b/source/Playnite/Database/GameDatabase.cs index 34ab1a36f..dc6876c42 100644 --- a/source/Playnite/Database/GameDatabase.cs +++ b/source/Playnite/Database/GameDatabase.cs @@ -1202,11 +1202,110 @@ public Game ImportGame(GameMetadata game, Guid pluginId) return toAdd; } - public List ImportGames(LibraryPlugin library, CancellationToken cancelToken, PlaytimeImportMode playtimeImportMode) + public class GameImportOperation + { + public enum OperationType { Add, Update } + public OperationType Type { get; set; } + public GameMetadata GameData { get; set; } + public Game ExistingGame { get; set; } + public Guid PluginId { get; set; } + public PlaytimeImportMode PlaytimeMode { get; set; } + } + + public class GameImportQueue + { + public List Operations { get; set; } = new List(); + public CompletionStatusSettings StatusSettings { get; set; } + } + + public GameImportQueue PrepareGameImports(LibraryPlugin library, CancellationToken cancelToken, PlaytimeImportMode playtimeImportMode) + { + var queue = new GameImportQueue + { + StatusSettings = GetCompletionStatusSettings() + }; + + if (library.Properties?.HasCustomizedGameImport == true) + { + // For custom import, we need to handle this differently as it bypasses GameMetadata + // This path will need to be handled in the main ImportGames method + return queue; + } + else + { + foreach (var newGame in library.GetGames(new LibraryGetGamesArgs { CancelToken = cancelToken }) ?? new List()) + { + if (ImportExclusions[ImportExclusionItem.GetId(newGame.GameId, library.Id)] != null) + { + logger.Debug($"Excluding {newGame.Name} {library.Name} from import."); + continue; + } + + var existingGame = Games.FirstOrDefault(a => a.GameId == newGame.GameId && a.PluginId == library.Id); + if (existingGame == null) + { + logger.Info(string.Format("Queuing new game {0} from {1} plugin for import", newGame.GameId, library.Name)); + queue.Operations.Add(new GameImportOperation + { + Type = GameImportOperation.OperationType.Add, + GameData = newGame, + PluginId = library.Id, + PlaytimeMode = playtimeImportMode + }); + } + else + { + // Check if update is needed + var needsUpdate = false; + if (!existingGame.IsCustomGame && !existingGame.OverrideInstallState) + { + if (existingGame.IsInstalled != newGame.IsInstalled || + !string.Equals(existingGame.InstallDirectory, newGame.InstallDirectory, StringComparison.OrdinalIgnoreCase)) + { + needsUpdate = true; + } + } + + if (playtimeImportMode == PlaytimeImportMode.Always && newGame.Playtime > 0) + { + if (existingGame.Playtime != newGame.Playtime || + (newGame.LastActivity != null && (existingGame.LastActivity == null || newGame.LastActivity > existingGame.LastActivity))) + { + needsUpdate = true; + } + } + + if (!existingGame.IsInstalled && newGame.InstallSize != null && newGame.InstallSize > 0 && + existingGame.InstallSize != newGame.InstallSize) + { + needsUpdate = true; + } + + if (needsUpdate) + { + queue.Operations.Add(new GameImportOperation + { + Type = GameImportOperation.OperationType.Update, + GameData = newGame, + ExistingGame = existingGame, + PluginId = library.Id, + PlaytimeMode = playtimeImportMode + }); + } + } + } + } + + return queue; + } + + public List ApplyGameImports(GameImportQueue queue) { using (BufferedUpdate()) { - var statusSettings = GetCompletionStatusSettings(); + var addedGames = new List(); + var statusSettings = queue.StatusSettings; + bool updateCompletionStatus(Game game, CompletionStatusSettings settings) { var updated = false; @@ -1226,59 +1325,38 @@ bool updateCompletionStatus(Game game, CompletionStatusSettings settings) return updated; } - if (library.Properties?.HasCustomizedGameImport == true) + foreach (var operation in queue.Operations) { - var importedGames = library.ImportGames(new LibraryImportGamesArgs { CancelToken = cancelToken })?.ToList() ?? new List(); - foreach (var game in importedGames) - { - updateCompletionStatus(game, statusSettings); - } - - return importedGames; - } - else - { - var addedGames = new List(); - foreach (var newGame in library.GetGames(new LibraryGetGamesArgs { CancelToken = cancelToken }) ?? new List()) + try { - if (ImportExclusions[ImportExclusionItem.GetId(newGame.GameId, library.Id)] != null) + if (operation.Type == GameImportOperation.OperationType.Add) { - logger.Debug($"Excluding {newGame.Name} {library.Name} from import."); - continue; - } + logger.Info(string.Format("Adding new game {0} from plugin {1}", operation.GameData.GameId, operation.PluginId)); - var existingGame = Games.FirstOrDefault(a => a.GameId == newGame.GameId && a.PluginId == library.Id); - if (existingGame == null) - { - logger.Info(string.Format("Adding new game {0} from {1} plugin", newGame.GameId, library.Name)); - try + if (operation.GameData.Playtime != 0) { - if (newGame.Playtime != 0) + var originalPlaytime = operation.GameData.Playtime; + operation.GameData.Playtime = 0; + if (operation.PlaytimeMode == PlaytimeImportMode.Always || + operation.PlaytimeMode == PlaytimeImportMode.NewImportsOnly) { - var originalPlaytime = newGame.Playtime; - newGame.Playtime = 0; - if (playtimeImportMode == PlaytimeImportMode.Always || - playtimeImportMode == PlaytimeImportMode.NewImportsOnly) - { - newGame.Playtime = originalPlaytime; - } - } - - var importedGame = ImportGame(newGame, library.Id); - addedGames.Add(importedGame); - if (updateCompletionStatus(importedGame, statusSettings)) - { - Games.Update(importedGame); + operation.GameData.Playtime = originalPlaytime; } } - catch (Exception e) when (!PlayniteEnvironment.ThrowAllErrors) + + var importedGame = ImportGame(operation.GameData, operation.PluginId); + addedGames.Add(importedGame); + if (updateCompletionStatus(importedGame, statusSettings)) { - logger.Error(e, "Failed to import game into database."); + Games.Update(importedGame); } } - else + else if (operation.Type == GameImportOperation.OperationType.Update) { + var existingGame = operation.ExistingGame; + var newGame = operation.GameData; var existingGameUpdated = false; + if (!existingGame.IsCustomGame && !existingGame.OverrideInstallState) { if (existingGame.IsInstalled != newGame.IsInstalled) @@ -1294,7 +1372,7 @@ bool updateCompletionStatus(Game game, CompletionStatusSettings settings) } } - if (playtimeImportMode == PlaytimeImportMode.Always && newGame.Playtime > 0) + if (operation.PlaytimeMode == PlaytimeImportMode.Always && newGame.Playtime > 0) { if (existingGame.Playtime != newGame.Playtime) { @@ -1302,9 +1380,6 @@ bool updateCompletionStatus(Game game, CompletionStatusSettings settings) existingGameUpdated = true; } - // The LastActivity value of the newGame is only applied if newer than - // the existing game, to prevent cases of DRM free games being launched without - // the client or offline, which would prevent the date from being updated in the service if (newGame.LastActivity != null && (existingGame.LastActivity == null || newGame.LastActivity > existingGame.LastActivity)) { @@ -1331,10 +1406,58 @@ bool updateCompletionStatus(Game game, CompletionStatusSettings settings) } } } + catch (Exception e) when (!PlayniteEnvironment.ThrowAllErrors) + { + logger.Error(e, "Failed to import game operation into database."); + } + } - return addedGames; + return addedGames; + } + } + + public List ImportGames(LibraryPlugin library, CancellationToken cancelToken, PlaytimeImportMode playtimeImportMode) + { + // Handle custom import separately as it bypasses the queue system + if (library.Properties?.HasCustomizedGameImport == true) + { + using (BufferedUpdate()) + { + var statusSettings = GetCompletionStatusSettings(); + bool updateCompletionStatus(Game game, CompletionStatusSettings settings) + { + var updated = false; + if ((game.Playtime > 0 && (game.CompletionStatusId == Guid.Empty || game.CompletionStatusId == settings.DefaultStatus)) && + game.CompletionStatusId != statusSettings.PlayedStatus) + { + game.CompletionStatusId = statusSettings.PlayedStatus; + updated = true; + } + else if ((game.Playtime == 0 && game.CompletionStatusId == Guid.Empty) && + game.CompletionStatusId != statusSettings.DefaultStatus) + { + game.CompletionStatusId = statusSettings.DefaultStatus; + updated = true; + } + + return updated; + } + + var importedGames = library.ImportGames(new LibraryImportGamesArgs { CancelToken = cancelToken })?.ToList() ?? new List(); + foreach (var game in importedGames) + { + updateCompletionStatus(game, statusSettings); + } + + return importedGames; } } + else + { + // Use the new split approach for standard imports + var queue = PrepareGameImports(library, cancelToken, playtimeImportMode); + return ApplyGameImports(queue); + } } public List GetSortedFilterPresets() @@ -1410,12 +1533,12 @@ public static void GenerateSampleData(IGameDatabase database) Playtime = 115200, LastActivity = DateTime.Today, IsInstalled = true, - AgeRatingIds = new List { database.AgeRatings.First().Id }, + AgeRatingIds = new List { database.AgeRatings.First().Id }, CategoryIds = new List { database.Categories.First().Id }, DeveloperIds = new List { database.Companies.First().Id }, PublisherIds = new List { database.Companies.Last().Id }, GenreIds = new List { database.Genres.First().Id }, - RegionIds = new List { database.Regions.First().Id }, + RegionIds = new List { database.Regions.First().Id }, SeriesIds = new List { database.Series.First().Id }, SourceId = database.Sources.First().Id, TagIds = new List { database.Tags.First().Id }, diff --git a/source/Playnite/ViewModels/MainViewModelBase.cs b/source/Playnite/ViewModels/MainViewModelBase.cs index 58b2a8709..65f567084 100644 --- a/source/Playnite/ViewModels/MainViewModelBase.cs +++ b/source/Playnite/ViewModels/MainViewModelBase.cs @@ -509,6 +509,117 @@ private List ImportLibraryGames(LibraryPlugin plugin, CancellationToken to return addedGames; } + private List ImportLibraryGamesMultithreaded(IEnumerable plugins, CancellationToken token) + { + var allAddedGames = new List(); + if (token.IsCancellationRequested || !plugins.Any()) + { + return allAddedGames; + } + + var pluginsList = plugins.ToList(); + Logger.Info($"Importing games from {pluginsList.Count} library plugins using multithreaded approach"); + ProgressStatus = Resources.GetString(LOC.ProgressImportinGames).Format($"{pluginsList.Count} libraries (multithreaded)"); + + try + { + // Phase 1: Prepare all game imports in parallel (thread-safe data collection) + var importQueues = new Dictionary(); + var preparePhaseProgress = 0; + var preparePhaseLock = new object(); + var preparePhaseErrors = new List(); + + var prepareOptions = new ParallelOptions + { + MaxDegreeOfParallelism = Math.Max(1, Math.Min(Environment.ProcessorCount, pluginsList.Count)), + CancellationToken = token + }; + + Parallel.ForEach(pluginsList, prepareOptions, plugin => + { + try + { + if (token.IsCancellationRequested) + { + return; + } + + Logger.Info($"Collecting games from {plugin.Name} (parallel)"); + var queue = ((GameDatabase)Database).PrepareGameImports(plugin, token, AppSettings.PlaytimeImportMode); + + lock (preparePhaseLock) + { + importQueues[plugin] = queue; + preparePhaseProgress++; + ProgressStatus = $"Collected data from {plugin.Name} ({preparePhaseProgress}/{pluginsList.Count})"; + } + + App.Notifications.Remove($"{plugin.Id} - download"); + } + catch (Exception ex) + { + lock (preparePhaseLock) + { + preparePhaseProgress++; + preparePhaseErrors.Add($"Failed to collect data from {plugin.Name}: {ex.Message}"); + Logger.Error(ex, $"Failed to collect data from {plugin.Name}"); + + App.Notifications.Add(new NotificationMessage( + $"{plugin.Id} - download", + Resources.GetString(LOC.LibraryImportError).Format(plugin.Name) + $"\n{ex.Message}", + NotificationType.Error)); + } + } + }); + + if (token.IsCancellationRequested) + { + return allAddedGames; + } + + // Phase 2: Apply all imports sequentially (database operations must be single-threaded) + Logger.Info($"Starting sequential database updates for {importQueues.Count} libraries"); + + int applyPhaseProgress = 0; + foreach (var kvp in importQueues) + { + if (token.IsCancellationRequested) + { + break; + } + + var plugin = kvp.Key; + var queue = kvp.Value; + + try + { + applyPhaseProgress++; + ProgressStatus = $"Applying updates from {plugin.Name} ({applyPhaseProgress}/{importQueues.Count})"; + Logger.Info($"Applying {queue.Operations.Count} operations from {plugin.Name}"); + + var games = ((GameDatabase)Database).ApplyGameImports(queue); + allAddedGames.AddRange(games); + + Logger.Info($"{plugin.Name}: {games.Count} games updated"); + } + catch (Exception ex) + { + Logger.Error(ex, $"Failed to apply updates from {plugin.Name}"); + App.Notifications.Add(new NotificationMessage( + $"{plugin.Id} - download", + Resources.GetString(LOC.LibraryImportError).Format(plugin.Name) + $"\n{ex.Message}", + NotificationType.Error)); + } + } + } + catch (Exception e) when (!PlayniteEnvironment.ThrowAllErrors) + { + Logger.Error(e, "Failed during multithreaded library import"); + } + + return allAddedGames; + } + public async Task ProcessStartupLibUpdate() { if (App.CmdLine.SkipLibUpdate) @@ -545,16 +656,8 @@ await UpdateLibraryData((token) => var addedGames = new List(); if (updateIntegrations) { - foreach (var plugin in Extensions.LibraryPlugins) - { - if (token.IsCancellationRequested) - { - return addedGames; - } - - addedGames.AddRange(ImportLibraryGames(plugin, token)); - } - + // Use multithreaded library import for better performance + addedGames.AddRange(ImportLibraryGamesMultithreaded(Extensions.LibraryPlugins, token)); AppSettings.LastLibraryUpdateCheck = DateTimes.Now; }