Added: Miscellaneous Bug Fixes around Updates

src/Networking/NexusMods.Networking.ModUpdates/PerFeedCacheUpdaterResult.cs

@@ -90,5 +90,6 @@ public enum CacheUpdaterResultStatus
     /// The result of this method call may contain a partial result, with the
     /// info obtained from before the call was rate limited.
     /// </summary>
+    // ReSharper disable once UnusedMember.Global
     RateLimited, // Placeholder.
 }

src/Networking/NexusMods.Networking.NexusWebApi/ModUpdateService.cs

@@ -6,6 +6,8 @@
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.Networking.ModUpdates;
 using NexusMods.Networking.ModUpdates.Mixins;
+using System;
+
 namespace NexusMods.Networking.NexusWebApi;
 
 /// <summary>
@@ -14,17 +16,19 @@ namespace NexusMods.Networking.NexusWebApi;
 /// </summary>
 public class ModUpdateService : IModUpdateService, IDisposable
 {
+    internal const int UpdateCheckCooldownSeconds = 30;
+
     private readonly IConnection _connection;
     private readonly INexusApiClient _nexusApiClient;
     private readonly IGameDomainToGameIdMappingCache _gameIdMappingCache;
     private readonly ILogger<ModUpdateService> _logger;
     private readonly NexusGraphQLClient _gqlClient;
+    private readonly TimeProvider _timeProvider;
 
     // Use SourceCache to maintain latest values per key
     private readonly SourceCache<KeyValuePair<NexusModsFileMetadataId, ModUpdateOnPage>, EntityId> _newestModVersionCache = new (static kv => kv.Key);
     private readonly SourceCache<KeyValuePair<NexusModsModPageMetadataId, ModUpdatesOnModPage>, EntityId> _newestModOnAnyPageCache = new (static kv => kv.Key);
     private readonly IDisposable _updateObserver;
-    private const int UpdateCheckCooldownSeconds = 30;
     private DateTimeOffset _lastUpdateCheckTime = DateTimeOffset.MinValue;
 
     /// <summary/>
@@ -33,13 +37,15 @@ public ModUpdateService(
         INexusApiClient nexusApiClient,
         IGameDomainToGameIdMappingCache gameIdMappingCache,
         ILogger<ModUpdateService> logger,
-        NexusGraphQLClient gqlClient)
+        NexusGraphQLClient gqlClient,
+        TimeProvider timeProvider)
     {
         _connection = connection;
         _nexusApiClient = nexusApiClient;
         _gameIdMappingCache = gameIdMappingCache;
         _logger = logger;
         _gqlClient = gqlClient;
+        _timeProvider = timeProvider;
         // Note(sewer): This is a singleton, so we don't actually need to dispose, that said
         // I'm opting to for the sake of following good practices.
         _updateObserver = ObserveUpdates();
@@ -84,7 +90,7 @@ private IDisposable ObserveUpdates()
 
                 // We grab entityID(s) because we want to query the DB for the
                 // latest info of the affected mod pages, not a possible snapshot.
-                var affectedModPage = new List<EntityId>();
+                var affectedModPage = new HashSet<EntityId>();
 
                 // Accept all events; Add, Update, Remove, Refresh, Reset
                 foreach (var change in changes)
@@ -109,17 +115,17 @@ public async Task<PerFeedCacheUpdaterResult<PageMetadataMixin>> CheckAndUpdateMo
         // _lastUpdateCheckTime is updated.
         if (throttle)
         {
-            var timeLeft = UpdateCheckCooldownSeconds - (int)(DateTimeOffset.UtcNow - _lastUpdateCheckTime).TotalSeconds;
+            var timeLeft = UpdateCheckCooldownSeconds - (int)(_timeProvider.GetUtcNow() - _lastUpdateCheckTime).TotalSeconds;
             if (timeLeft > 0)
             {
                 _logger.LogInformation("Skipping update check due to rate limit ({cooldown} seconds). Time left: {timeLeft} seconds.", UpdateCheckCooldownSeconds, timeLeft);
-                return PerFeedCacheUpdaterResult<PageMetadataMixin>.WithStatus(CacheUpdaterResultStatus.RateLimited);
+                return PerFeedCacheUpdaterResult<PageMetadataMixin>.WithStatus(CacheUpdaterResultStatus.Throttled);
             }
-            _lastUpdateCheckTime = DateTimeOffset.UtcNow;
+            _lastUpdateCheckTime = _timeProvider.GetUtcNow();
         }
         else
         {
-            _lastUpdateCheckTime = DateTimeOffset.UtcNow;
+            _lastUpdateCheckTime = _timeProvider.GetUtcNow();
         }
 
         // Identify all mod pages needing a refresh
@@ -158,26 +164,39 @@ public void NotifyForUpdates()
             .DistinctBy(static fileMetadata => fileMetadata.Id)
             .ToDictionary(static x => x.Id, static x => x);
 
-        NotifyForUpdatesOfSpecificFiles(filesInLibrary);
+        NotifyForUpdatesOfSpecificFiles(filesInLibrary, UpdateNewestModVersionCache, UpdateNewestModOnAnyPageCache);
     }
 
-    private void NotifyForUpdatesOfSpecificFiles(Dictionary<EntityId, NexusModsFileMetadata.ReadOnly> filesInLibrary)
+    private void NotifyForUpdatesOfSpecificFiles(
+        Dictionary<EntityId, NexusModsFileMetadata.ReadOnly> filesInLibrary, 
+        ModVersionCacheUpdateDelegate modVersionUpdater, 
+        ModPageCacheUpdateDelegate modPageUpdater)
     {
         var existingFileToNewerFiles = filesInLibrary
             .Select(kv =>
             {
                 var newerFiles = RunUpdateCheck
                     .GetNewerFilesForExistingFile(kv.Value)
-                    // `!filesInLibrary.ContainsKey(newFile.Id)`: This filters out the case where we already have the latest file version on disk. 
-                    .Where(newFile => newFile.IsValid() && !filesInLibrary.ContainsKey(newFile.Id))
+                    .Where(newFile => newFile.IsValid())
                     .ToArray();
 
-                return new ModUpdateOnPage(kv.Value, newerFiles);
+                var hasUpdate = newerFiles.Length switch
+                {
+                    // If the newest item for this mod is not in the library, we have an update.
+                    > 0 => !filesInLibrary.ContainsKey(newerFiles[0].Id),
+                    <= 0 => false,
+                };
+
+                return hasUpdate 
+                    ? new ModUpdateOnPage(kv.Value, newerFiles)
+                    // If there is no update, then return default struct, which will have
+                    // a 0 number of new files, and thus not be a valid update mapping.
+                    : new ModUpdateOnPage(default(NexusModsFileMetadata.ReadOnly),[]);
             })
             .Where(static kv => kv.NewerFiles.Length > 0)
             .ToDictionary(page => page.File.Id);
 
-        UpdateNewestModVersionCache(existingFileToNewerFiles);
+        modVersionUpdater(existingFileToNewerFiles);
 
         var modPageToNewerFiles = existingFileToNewerFiles
             .GroupBy(
@@ -189,7 +208,7 @@ private void NotifyForUpdatesOfSpecificFiles(Dictionary<EntityId, NexusModsFileM
                 group => (ModUpdatesOnModPage)group.ToArray()
             );
 
-        UpdateNewestModOnAnyPageCache(modPageToNewerFiles);
+        modPageUpdater(modPageToNewerFiles);
     }
 
     /// <inheritdoc />
@@ -216,6 +235,10 @@ public void Dispose()
         _updateObserver.Dispose();
     }
 
+    /// <summary>
+    /// Updates the internal state of the <see cref="_newestModVersionCache"/> such that it
+    /// matches the contents of <paramref name="existingFileToNewerFiles"/>.
+    /// </summary>
     private void UpdateNewestModVersionCache(Dictionary<EntityId, ModUpdateOnPage> existingFileToNewerFiles)
     {
         // First remove any invalid files, and then add any newer files.
@@ -235,6 +258,40 @@ private void UpdateNewestModVersionCache(Dictionary<EntityId, ModUpdateOnPage> e
         });
     }
 
+    /// <summary>
+    /// <see cref="UpdateNewestModVersionCache"/> but scoped to a limited set of mod pages
+    /// marked by <paramref name="affectedModPageIds"/>. Anything not related to those mod
+    /// pages is not touched.
+    /// </summary>
+    /// <remarks>
+    ///     This is run on the hot path that reacts to changes in library.
+    /// </remarks>
+    private void UpdateNewestModVersionCachePartial(Dictionary<EntityId, ModUpdateOnPage> existingFileToNewerFiles, HashSet<EntityId> affectedModPageIds)
+    {
+        // First remove any invalid files, and then add any newer files.
+        _newestModVersionCache.Edit(updater =>
+        {
+            foreach (var kv in updater.Items)
+            {
+                // Check if this file belongs to any of the affected mod pages.
+                // We shouldn't remove updates from any other mod pages.
+                if (!affectedModPageIds.Contains(kv.Value.File.ModPageId)) continue;
+
+                // Entries not in our partial set filtered by mod page should be removed.
+                if (!existingFileToNewerFiles.ContainsKey(kv.Key))
+                    updater.Remove(kv.Key);
+            }
+
+            // Add any newer files/items.
+            foreach (var kv in existingFileToNewerFiles)
+                updater.AddOrUpdate(new KeyValuePair<NexusModsFileMetadataId, ModUpdateOnPage>(kv.Key, kv.Value));
+        });
+    }
+
+    /// <summary>
+    /// Updates the internal state of the <see cref="_newestModOnAnyPageCache"/> such that it
+    /// matches the contents of <paramref name="modPageToNewerFiles"/>.
+    /// </summary>
     private void UpdateNewestModOnAnyPageCache(Dictionary<NexusModsModPageMetadataId, ModUpdatesOnModPage> modPageToNewerFiles)
     {
         // First remove any invalid mod pages, and then add any newer files.
@@ -253,8 +310,40 @@ private void UpdateNewestModOnAnyPageCache(Dictionary<NexusModsModPageMetadataId
                 updater.AddOrUpdate(kv);
         });
     }
+
+    /// <summary>
+    /// <see cref="UpdateNewestModOnAnyPageCache"/> but scoped to a limited set of mod pages
+    /// marked by <paramref name="affectedModPageIds"/>. Anything not related to those mod
+    /// pages is not touched.
+    /// </summary>
+    /// <remarks>
+    ///     This is run on the hot path that reacts to changes in library.
+    /// </remarks>
+    private void UpdateNewestModOnAnyPageCachePartial(Dictionary<NexusModsModPageMetadataId, ModUpdatesOnModPage> modPageToNewerFiles, HashSet<EntityId> affectedModPageIds)
+    {
+        // First remove any invalid mod pages, and then add any newer files.
+        _newestModOnAnyPageCache.Edit(updater =>
+        {
+            // Remove any currently known mod pages from _newestModOnAnyPageCache
+            // that no longer require to be updated.
+            foreach (var existingKey in updater.Keys)
+            {
+                // Check if this mod page is one of the affected ones.
+                // Pages not in set to be updated should be ignored.
+                if (!affectedModPageIds.Contains(existingKey)) continue;
 
-    private void NotifyForUpdatesOfSpecificModPages(List<EntityId> modPageIds)
+                // Only remove after filter if it's no longer in the updated list
+                if (!modPageToNewerFiles.ContainsKey(existingKey))
+                    updater.Remove(existingKey);
+            }
+
+            // Add any newer pages
+            foreach (var kv in modPageToNewerFiles)
+                updater.AddOrUpdate(kv);
+        });
+    }
+
+    private void NotifyForUpdatesOfSpecificModPages(HashSet<EntityId> modPageIds)
     {
         // Note(sewer): Change to generic inheriting IEnumerable if ever making this public.
         var filesInLibrary = new Dictionary<EntityId, NexusModsFileMetadata.ReadOnly>();
@@ -284,8 +373,13 @@ private void NotifyForUpdatesOfSpecificModPages(List<EntityId> modPageIds)
         }
 
         // And now beam the update stuff, brrr!!
-        NotifyForUpdatesOfSpecificFiles(filesInLibrary);
+        NotifyForUpdatesOfSpecificFiles(filesInLibrary, 
+            existingFileToNewerFiles=> UpdateNewestModVersionCachePartial(existingFileToNewerFiles, modPageIds), 
+            modPageToNewerFiles=> UpdateNewestModOnAnyPageCachePartial(modPageToNewerFiles, modPageIds));
     }
+
+    private delegate void ModVersionCacheUpdateDelegate(Dictionary<EntityId, ModUpdateOnPage> cache);
+    private delegate void ModPageCacheUpdateDelegate(Dictionary<NexusModsModPageMetadataId, ModUpdatesOnModPage> cache);
 }
 
 /// <summary>
@@ -302,7 +396,7 @@ public readonly record struct ModUpdatesOnModPage(ModUpdateOnPage[] FileMappings
     /// Given that each array entry represents a single mod file, this is just the count of the internal array.
     /// </summary>
     public int NumberOfModFilesToUpdate => FileMappings.Length;
-    
+
     /// <summary>
     /// Returns the newest file across mods on this mod page.
     /// </summary>
@@ -326,7 +420,7 @@ public NexusModsFileMetadata.ReadOnly NewestFile()
 
         return newestFile;
     }
-    
+
     /// <summary>
     /// Returns the newest file from every mod on this page.
     /// See Remarks before use.

src/Networking/NexusMods.Networking.NexusWebApi/Services.cs

@@ -44,6 +44,7 @@ public static IServiceCollection AddNexusWebApi(this IServiceCollection collecti
         collection.AddApiKeyModel();
         collection.AddGameDomainToGameIdMappingModel();
         collection.AddAllSingleton<IGameDomainToGameIdMappingCache, GameDomainToGameIdMappingCache>();
+        collection.AddSingleton(TimeProvider.System);
 
         collection
             .AddNexusModsLibraryModels()