Merge pull request #600 from osu-tournament-rating/refactor/data-parser

rewrite the data parser to support new beatmap structure

.github/workflows/ci-api.yml

@@ -21,11 +21,11 @@ jobs:
               run: dotnet restore
 
             - name: .NET Warnings & Code Style
-              run: dotnet build -c Debug -warnaserror -p:EnforceCodeStyleInBuild=true
+              run: dotnet build -c Debug -warnaserror
 
             - name: .NET Format
               run: |
-                dotnet format --severity info --exclude **/Migrations/* --verify-no-changes
+                dotnet format --severity info --exclude **/Migrations/* --verify-no-changes --verbosity diagnostic
     
     test:
       name: Test

API/Controllers/UsersController.cs

@@ -55,7 +55,7 @@ public async Task<IActionResult> GetAsync(int id)
     [ProducesResponseType<UserDTO>(StatusCodes.Status200OK)]
     public async Task<IActionResult> UpdateScopesAsync(int id, [FromBody][Required] List<string> scopes)
     {
-        scopes = scopes.Select(s => s.ToLower()).ToList();
+        scopes = [.. scopes.Select(s => s.ToLower())];
         foreach (var scope in scopes)
         {
             if (!OtrClaims.Roles.IsUserAssignableRole(scope))

API/Program.cs

@@ -338,7 +338,7 @@ await context.HttpContext.Response.WriteAsync(
         {
             Type = "string",
             Description = "The possible roles assignable to a user or client",
-            Enum = new List<IOpenApiAny>(oauthScopes.Keys.Select(role => new OpenApiString(role))),
+            Enum = [.. oauthScopes.Keys.Select(role => new OpenApiString(role))],
             Extensions = new Dictionary<string, IOpenApiExtension>
             {
                 [ExtensionKeys.EnumNames] =

API/Repositories/Implementations/OAuthClientRepository.cs

@@ -46,6 +46,6 @@ public string GenerateClientSecret()
         const string chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
 
         var r = new Random();
-        return new string(Enumerable.Repeat(chars, length).Select(s => s[r.Next(s.Length)]).ToArray());
+        return new string([.. Enumerable.Repeat(chars, length).Select(s => s[r.Next(s.Length)])]);
     }
 }

API/Services/Implementations/SearchService.cs

@@ -18,9 +18,9 @@ ICacheHandler cacheHandler
     public async Task<SearchResponseCollectionDTO> SearchByNameAsync(string searchKey) =>
         new()
         {
-            Tournaments = (await SearchTournamentsByNameAsync(searchKey)).ToList(),
-            Matches = (await SearchMatchesByNameAsync(searchKey)).ToList(),
-            Players = (await SearchPlayersByNameAsync(searchKey)).ToList()
+            Tournaments = [.. (await SearchTournamentsByNameAsync(searchKey))],
+            Matches = [.. (await SearchMatchesByNameAsync(searchKey))],
+            Players = [.. (await SearchPlayersByNameAsync(searchKey))]
         };
 
     private async Task<IEnumerable<TournamentSearchResultDTO>> SearchTournamentsByNameAsync(string tournamentName)
@@ -34,7 +34,7 @@ await cacheHandler.Cache.GetObjectAsync<IEnumerable<TournamentSearchResultDTO>>(
             return result;
         }
 
-        result = (await tournamentsRepository.SearchAsync(tournamentName)).ToList();
+        result = [.. (await tournamentsRepository.SearchAsync(tournamentName))];
         await cacheHandler.SetTournamentSearchResultAsync(result, tournamentName);
 
         return result;
@@ -51,7 +51,7 @@ await cacheHandler.Cache.GetObjectAsync<IEnumerable<MatchSearchResultDTO>>(
             return result;
         }
 
-        result = (await matchesService.SearchAsync(matchName)).ToList();
+        result = [.. (await matchesService.SearchAsync(matchName))];
         await cacheHandler.SetMatchSearchResultAsync(result, matchName);
 
         return result;
@@ -68,7 +68,7 @@ private async Task<IEnumerable<PlayerSearchResultDTO>> SearchPlayersByNameAsync(
             return result;
         }
 
-        result = (await playerRepository.SearchAsync(username))
+        result = [.. (await playerRepository.SearchAsync(username))
             .Select(player =>
             {
                 PlayerRating? stats = player.Ratings
@@ -82,8 +82,7 @@ private async Task<IEnumerable<PlayerSearchResultDTO>> SearchPlayersByNameAsync(
                     Username = player.Username,
                     Thumbnail = $"a.ppy.sh/{player.OsuId}"
                 };
-            })
-            .ToList();
+            })];
         await cacheHandler.SetPlayerSearchResultAsync(result, username);
 
         return result;

API/Services/Implementations/TournamentsService.cs

@@ -43,10 +43,9 @@ bool preApprove
                 ? TournamentProcessingStatus.NeedsMatchData
                 : TournamentProcessingStatus.NeedsApproval,
             SubmittedByUserId = submitterUserId,
-            Matches = submittedMatchIds
+            Matches = [.. submittedMatchIds
                 .Except(existingMatchIds)
-                .Select(matchId => new Match { OsuId = matchId, SubmittedByUserId = submitterUserId })
-                .ToList(),
+                .Select(matchId => new Match { OsuId = matchId, SubmittedByUserId = submitterUserId })],
             PooledBeatmaps =
 
             [

API/Services/Implementations/UserService.cs

@@ -50,7 +50,7 @@ public async Task<bool> RejectSubmissionsAsync(int id, int? rejecterUserId)
             return null;
         }
 
-        user.Scopes = scopes.ToArray();
+        user.Scopes = [.. scopes];
         await userRepository.UpdateAsync(user);
 
         return mapper.Map<UserDTO>(user);

API/SwaggerGen/Filters/EnumMetadataSchemaFilter.cs

@@ -17,10 +17,9 @@ namespace API.SwaggerGen.Filters;
 [UsedImplicitly]
 public class EnumMetadataSchemaFilter(string[] xmlDocPaths) : ISchemaFilter
 {
-    private readonly IList<XPathNavigator> _xmlNavigators = xmlDocPaths
+    private readonly IList<XPathNavigator> _xmlNavigators = [.. xmlDocPaths
         .Where(File.Exists)
-        .Select(path => new XPathDocument(path).CreateNavigator())
-        .ToList();
+        .Select(path => new XPathDocument(path).CreateNavigator())];
 
     public void Apply(OpenApiSchema schema, SchemaFilterContext context)
     {

API/SwaggerGen/Filters/SecurityMetadataOperationFilter.cs

@@ -22,9 +22,7 @@ public void Apply(OpenApiOperation operation, OperationFilterContext context)
             return;
         }
 
-        IEnumerable<AuthorizeAttribute> authAttributes = context
-            .GetControllerAndActionAttributes<AuthorizeAttribute>()
-            .ToList();
+        IEnumerable<AuthorizeAttribute> authAttributes = [.. context.GetControllerAndActionAttributes<AuthorizeAttribute>()];
         if (!authAttributes.Any())
         {
             operation.AddAuthExtension(false);

DataWorkerService/AutomationChecks/Matches/MatchHeadToHeadCheck.cs

@@ -46,9 +46,7 @@ g.VerificationStatus is VerificationStatus.PreRejected
 
         logger.LogInformation("Attempting to convert HeadToHead games to TeamVs [Id: {Id}]", entity.Id);
 
-        IEnumerable<Game> preRejectedGames = entity.Games
-            .Where(g => g.VerificationStatus is VerificationStatus.PreRejected)
-            .ToList();
+        IEnumerable<Game> preRejectedGames = [.. entity.Games.Where(g => g.VerificationStatus is VerificationStatus.PreRejected)];
 
         // Decide which players are Red and Blue
         var firstGameScores = preRejectedGames.First().Scores

DataWorkerService/Processors/Games/GameStatsProcessor.cs

@@ -74,8 +74,8 @@ public static GameWinRecord GenerateWinRecord(IEnumerable<GameScore> scores)
         {
             WinnerTeam = winningTeam,
             LoserTeam = losingTeam,
-            WinnerRoster = eScores.Where(s => s.Team == winningTeam).Select(s => s.PlayerId).ToArray(),
-            LoserRoster = eScores.Where(s => s.Team == losingTeam).Select(s => s.PlayerId).ToArray(),
+            WinnerRoster = [.. eScores.Where(s => s.Team == winningTeam).Select(s => s.PlayerId)],
+            LoserRoster = [.. eScores.Where(s => s.Team == losingTeam).Select(s => s.PlayerId)],
             WinnerScore = eScores.Where(s => s.Team == winningTeam).Sum(s => s.Score),
             LoserScore = eScores.Where(s => s.Team == losingTeam).Sum(s => s.Score)
         };

DataWorkerService/Processors/Matches/MatchStatsProcessor.cs

@@ -48,9 +48,7 @@ protected override async Task OnProcessingAsync(Match entity, CancellationToken
             }
         }
 
-        IEnumerable<Game> verifiedGames = entity.Games
-            .Where(g => g is { VerificationStatus: VerificationStatus.Verified, ProcessingStatus: GameProcessingStatus.Done })
-            .ToList();
+        IEnumerable<Game> verifiedGames = [.. entity.Games.Where(g => g is { VerificationStatus: VerificationStatus.Verified, ProcessingStatus: GameProcessingStatus.Done })];
 
         // Sanity check
         foreach (Game game in verifiedGames)
@@ -71,7 +69,7 @@ game.WinRecord is not null
         }
 
         entity.WinRecord = GenerateWinRecord(verifiedGames);
-        entity.PlayerMatchStats = GeneratePlayerMatchStats(verifiedGames).ToList();
+        entity.PlayerMatchStats = [.. GeneratePlayerMatchStats(verifiedGames)];
 
         entity.ProcessingStatus = MatchProcessingStatus.NeedsRatingProcessorData;
     }
@@ -108,16 +106,14 @@ public static MatchWinRecord GenerateWinRecord(IEnumerable<Game> games)
         {
             WinnerTeam = winningTeam,
             LoserTeam = losingTeam,
-            WinnerRoster = eGames
+            WinnerRoster = [.. eGames
                 .Select(g => g.WinRecord)
                 .SelectMany(gwr => gwr!.WinnerTeam == winningTeam ? gwr.WinnerRoster : gwr.LoserRoster)
-                .Distinct()
-                .ToArray(),
-            LoserRoster = eGames
+                .Distinct()],
+            LoserRoster = [.. eGames
                 .Select(g => g.WinRecord)
                 .SelectMany(gwr => gwr!.WinnerTeam == losingTeam ? gwr.WinnerRoster : gwr.LoserRoster)
-                .Distinct()
-                .ToArray(),
+                .Distinct()],
             WinnerScore = eGames.Count(g => g.WinRecord!.WinnerTeam == winningTeam),
             LoserScore = eGames.Count(g => g.WinRecord!.WinnerTeam == losingTeam),
         };
@@ -188,16 +184,14 @@ public static IEnumerable<PlayerMatchStats> GeneratePlayerMatchStats(IEnumerable
                     GamesLost = scores.Count(s => s.Game.WinRecord!.LoserRoster.Contains(playerId)),
                     Won = scores.First().Game.Match.WinRecord!.WinnerRoster.Contains(playerId),
                     // Reusing score groupings to ensure score filtering
-                    TeammateIds = playerScoreGroups
+                    TeammateIds = [.. playerScoreGroups
                         .Where(g => g.Item2.All(s => s.Team == scores.First().Team))
                         .Select(g => g.Key)
-                        .Where(id => id != playerId)
-                        .ToArray(),
-                    OpponentIds = playerScoreGroups
+                        .Where(id => id != playerId)],
+                    OpponentIds = [.. playerScoreGroups
                         .Where(g => g.Item2.All(s => s.Team != scores.First().Team))
                         .Select(g => g.Key)
-                        .Where(id => id != playerId)
-                        .ToArray(),
+                        .Where(id => id != playerId)],
                     PlayerId = playerId
                 };
             });

DataWorkerService/Processors/Resolvers/Implementations/ProcessorResolver.cs

@@ -9,7 +9,7 @@ public abstract class ProcessorResolver<TEntity> : IProcessorResolver<TEntity> w
 
     protected ProcessorResolver(IEnumerable<IProcessor<TEntity>> processors)
     {
-        processors = processors.ToList();
+        processors = [.. processors];
         if (!processors.Any())
         {
             throw new InvalidOperationException($"No processors were registered [Entity: {nameof(TEntity)}]");

DataWorkerService/Processors/Tournaments/TournamentDataProcessor.cs

@@ -2,9 +2,6 @@
 using Database.Enums.Verification;
 using DataWorkerService.Processors.Resolvers.Interfaces;
 using DataWorkerService.Services.Interfaces;
-using OsuApiClient;
-using OsuApiClient.Domain.Osu.Beatmaps;
-using Beatmap = Database.Entities.Beatmap;
 
 namespace DataWorkerService.Processors.Tournaments;
 
@@ -14,7 +11,6 @@ namespace DataWorkerService.Processors.Tournaments;
 public class TournamentDataProcessor(
     ILogger<TournamentDataProcessor> logger,
     IMatchProcessorResolver matchProcessorResolver,
-    IOsuClient osuClient,
     IOsuApiDataParserService osuApiDataParserService
 ) : ProcessorBase<Tournament>(logger)
 {
@@ -31,25 +27,14 @@ protected override async Task OnProcessingAsync(Tournament entity, CancellationT
             return;
         }
 
+        await osuApiDataParserService.ProcessBeatmapsAsync(entity.PooledBeatmaps.Select(b => b.OsuId));
+
         IProcessor<Match> matchDataProcessor = matchProcessorResolver.GetDataProcessor();
         foreach (Match match in entity.Matches)
         {
             await matchDataProcessor.ProcessAsync(match, cancellationToken);
         }
 
-        // Process data for pooled beatmaps that were not played
-        foreach (Beatmap beatmap in entity.PooledBeatmaps.Where(b => b.Games.Count == 0 && !b.HasData))
-        {
-            BeatmapExtended? apiBeatmap = await osuClient.GetBeatmapAsync(beatmap.OsuId, cancellationToken);
-
-            if (apiBeatmap is null)
-            {
-                continue;
-            }
-
-            await osuApiDataParserService.ParseBeatmap(beatmap, apiBeatmap);
-        }
-
         logger.LogInformation(
             "Tournament data processing summary " +
             "[Matches: {MCnt} | Games: {GCnt} | Beatmaps: {BCnt} | Scores: {SCnt} | Players: {PCnt}]",

DataWorkerService/Processors/Tournaments/TournamentStatsProcessor.cs

@@ -46,10 +46,9 @@ protected override async Task OnProcessingAsync(Tournament entity, CancellationT
             }
         }
 
-        IEnumerable<Match> verifiedMatches = entity.Matches
+        IEnumerable<Match> verifiedMatches = [.. entity.Matches
             .Where(m => m is
-            { VerificationStatus: VerificationStatus.Verified, ProcessingStatus: MatchProcessingStatus.Done })
-            .ToList();
+            { VerificationStatus: VerificationStatus.Verified, ProcessingStatus: MatchProcessingStatus.Done })];
 
         // Sanity check
         // If any processor data or stat objects are missing we cannot generate tournament stats
@@ -85,15 +84,13 @@ match.WinRecord is not null
                      .DistinctBy(p => p.Id)
                 )
         {
-            IEnumerable<PlayerMatchStats> matchStats = verifiedMatches
+            IEnumerable<PlayerMatchStats> matchStats = [.. verifiedMatches
                 .SelectMany(m => m.PlayerMatchStats)
-                .Where(pms => pms.Player.Id == player.Id)
-                .ToList();
+                .Where(pms => pms.Player.Id == player.Id)];
 
-            IEnumerable<RatingAdjustment> matchAdjustments = verifiedMatches
+            IEnumerable<RatingAdjustment> matchAdjustments = [.. verifiedMatches
                 .SelectMany(m => m.PlayerRatingAdjustments)
-                .Where(ra => ra.Player.Id == player.Id)
-                .ToList();
+                .Where(ra => ra.Player.Id == player.Id)];
 
             entity.PlayerTournamentStats.Add(new PlayerTournamentStats
             {
@@ -108,7 +105,7 @@ match.WinRecord is not null
                 GamesPlayed = matchStats.Sum(pms => pms.GamesPlayed),
                 GamesWon = matchStats.Sum(pms => pms.GamesWon),
                 GamesLost = matchStats.Sum(pms => pms.GamesLost),
-                TeammateIds = matchStats.SelectMany(pms => pms.TeammateIds).Distinct().ToArray(),
+                TeammateIds = [.. matchStats.SelectMany(pms => pms.TeammateIds).Distinct()],
                 PlayerId = player.Id,
                 TournamentId = entity.Id
             });