Fix ComboRatings GetComboDsstatsCalcDtos

src/dsstats.api/appsettings.Development.json

@@ -3,7 +3,7 @@
     "LogLevel": {
       "Default": "Information",
       "Microsoft.AspNetCore": "Warning",
-      "Microsoft.EntityFrameworkCore": "Information",
+      "Microsoft.EntityFrameworkCore": "Warning",
       "System.Net.Http": "Warning"
     }
   }

src/dsstats.ratings/ComboRatings.ArcadeRep.cs

@@ -1,5 +1,6 @@
 using dsstats.shared.Calc;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
 
 namespace dsstats.ratings;
 
@@ -12,6 +13,8 @@ public partial class ComboRatings
 
     private async Task InitArcadeRep(List<CalcDto> dsstatsReplays)
     {
+        // await CreateMaterializedReplays();
+
         var oldestReplayDate = dsstatsReplays.First().GameTime.AddDays(-dateOverflow);
         var latestReplayDate = dsstatsReplays.Last().GameTime.AddDays(dateOverflow);
 
@@ -49,7 +52,7 @@ private async Task CreateChunks(DateTime oldestReplayDate, DateTime latestReplay
                 StartTime = stepDate,
                 StartId = stepStartId
             };
-            stepDate = stepDate.AddDays(10);
+            stepDate = stepDate.AddDays(15);
             int stepEndId;
             if (stepDate > latestReplayDate)
             {
@@ -65,6 +68,7 @@ private async Task CreateChunks(DateTime oldestReplayDate, DateTime latestReplay
             chunkInfos.Add(stepChunkInfo);
             stepStartId = stepEndId + 1;
         }
+        logger.LogInformation("Got {count} chunks", chunkInfos.Count);
     }
 
     private async Task<int> GetStartIdAsync(DateTime date)
@@ -101,11 +105,13 @@ private async Task LoadCurrentChunkInfoArcadeReplays()
                 preserveCalcDtos = currentArcadeCalcDtos.Skip(skip).ToList();
             }
         }
-
         currentArcadeCalcDtos = await GetComboArcadeCalcDtos(chunkInfos[currentChunkInfoIndex]);
         currentArcadeCalcDtos = currentArcadeCalcDtos
             .Where(x => !matchesInfo.ArcadeDict.ContainsKey(x.ReplayId))
             .ToList();
+        logger.LogInformation("Loaded chunk {index}/{count}", currentChunkInfoIndex, currentArcadeCalcDtos.Count);
+        logger.LogInformation(chunkInfos[currentChunkInfoIndex].ToString());
+
         if (preserveCalcDtos.Count > 0)
         {
             currentArcadeCalcDtos = preserveCalcDtos.Concat(currentArcadeCalcDtos).ToList();
@@ -123,6 +129,10 @@ private async Task UpdateArcadeReplays(CalcDto dsstasReplay)
                 currentChunkInfo = chunkInfos[currentChunkInfoIndex];
                 await LoadCurrentChunkInfoArcadeReplays();
             }
+            else
+            {
+                logger.LogWarning("currentChunkInfoIndex out of bounds: {current}/{count}", currentChunkInfoIndex, chunkInfos.Count);
+            }
         }
     }
 

src/dsstats.ratings/ComboRatings.cs

@@ -1,13 +1,16 @@
 using dsstats.db8;
+using dsstats.shared;
 using dsstats.shared.Calc;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using MySqlConnector;
 using System.Diagnostics;
 using System.Text.Json;
 
 namespace dsstats.ratings;
 
-public partial class ComboRatings(ReplayContext context, ILogger<ComboRatings> logger)
+public partial class ComboRatings(ReplayContext context, IOptions<DbImportOptions> dbOptions, ILogger<ComboRatings> logger)
 {
     private Dictionary<ReplayKey, int> RegionDict = [];
     private Dictionary<ReplayKey, List<int>> ToonIdsDict = [];
@@ -37,17 +40,19 @@ public async Task CombineDsstatsSc2ArcadeReplays()
             FromDate = new DateTime(2021, 2, 1),
             GameModes = new List<int>() { 3, 4, 7 },
             Skip = 0,
-            Take = 1000
+            Take = 5000
         };
         HashSet<int> matchedArcadeIds = [];
         matchesInfo = await GetProcessedReplayIds();
         dsstatsRequest.Imported = dsstatsRequest.Imported == DateTime.MinValue ? DateTime.MinValue
             : matchesInfo.LatestUpdate.AddDays(-0.5);
         int matches = 0;
         var dsstatsReplays = await GetComboDsstatsCalcDtos(dsstatsRequest, context);
-
+        int dsstatsReplaysCount = dsstatsReplays.Count;
+        int i = 0;
         while (dsstatsReplays.Count > 0)
         {
+            i++;
             dsstatsReplays = dsstatsReplays.Where(x => !matchesInfo.ReplayDict.ContainsKey(x.ReplayId))
                 .ToList();
             await InitArcadeRep(dsstatsReplays);
@@ -72,13 +77,23 @@ public async Task CombineDsstatsSc2ArcadeReplays()
                 }
             }
             await StoreReplayMatches(replayMatches);
+            // if (i > 19)
+            // {
+            //     break;
+            // }
+            logger.LogInformation("Matches: {matches}/{total} ({percentage}%)", matches,
+                dsstatsReplaysCount, dsstatsReplaysCount > 0 ? Math.Round(matches * 100.0 / (double)dsstatsReplaysCount, 2) : 0);
+
             dsstatsRequest.Skip += dsstatsRequest.Take;
             dsstatsReplays = await GetComboDsstatsCalcDtos(dsstatsRequest, context);
+            dsstatsReplaysCount += dsstatsReplays.Count;
             RegionDict.Clear();
             ToonIdsDict.Clear();
         }
         sw.Stop();
-        logger.LogWarning("Matches: {matches} in {elapsed}sec", matches, Math.Round(sw.Elapsed.TotalSeconds, 2));
+        logger.LogWarning("Matches: {matches}/{total} ({percentage}%) in {elapsed}sec", matches,
+            dsstatsReplaysCount, dsstatsReplaysCount > 0 ? Math.Round(matches * 100.0 / (double)dsstatsReplaysCount, 2) : 0,
+             Math.Round(sw.Elapsed.TotalSeconds, 2));
     }
 
 
@@ -194,6 +209,7 @@ private int GetReplayRegionId(CalcDto replay)
 
     private static async Task<List<CalcDto>> GetComboDsstatsCalcDtos(DsstatsCalcRequest request, ReplayContext context)
     {
+        var noImportedDate = request.Imported == DateTime.MinValue;
         var query = from r in context.Replays
                     join m in context.ReplayArcadeMatches on r.ReplayId equals m.ReplayId into grouping
                     from m in grouping.DefaultIfEmpty()
@@ -204,8 +220,7 @@ from m in grouping.DefaultIfEmpty()
                         && request.GameModes.Contains((int)r.GameMode)
                         && r.TournamentEdition == false
                         && r.GameTime >= request.FromDate
-                        && (request.Continue ? r.ReplayRatingInfo == null : true)
-                        && (r.Imported == DateTime.MinValue || r.Imported > request.Imported)
+                        && (noImportedDate || r.Imported > request.Imported)
                     orderby r.GameTime, r.ReplayId
                     select new RawCalcDto
                     {
@@ -237,6 +252,26 @@ from m in grouping.DefaultIfEmpty()
 
         return rawDtos.Select(s => s.GetCalcDto()).ToList();
     }
+
+    private async Task CreateMaterializedReplays()
+    {
+        try
+        {
+
+            using var connection = new MySqlConnection(dbOptions.Value.ImportConnectionString);
+            await connection.OpenAsync();
+
+            var command = connection.CreateCommand();
+            command.CommandTimeout = 120;
+            command.CommandText = "CALL CreateMaterializedArcadeReplays();";
+
+            await command.ExecuteNonQueryAsync();
+        }
+        catch (Exception ex)
+        {
+            logger.LogError("failed creating materialized arcade replays: {error}", ex.Message);
+        }
+    }
 }
 
 internal record ReplayKey

src/dsstats.ratings/Program.cs

@@ -95,6 +95,12 @@ static void Main(string[] args)
                 SetLastSpawnHashes(serviceProvider);
                 CheckLastSpawnHashDuplicates(serviceProvider);
             }
+            else if (args[0] == "prep")
+            {
+                logger.LogInformation("CombineDsstatsSc2ArcadeReplays");
+                var comboRatings = scope.ServiceProvider.GetRequiredService<ComboRatings>();
+                comboRatings.CombineDsstatsSc2ArcadeReplays().Wait();
+            }
             else
             {
                 logger.LogError("allowed parameters: dsstats|sc2arcade|combo");