diff --git a/starsky/starsky.feature.thumbnail/Services/DatabaseThumbnailGenerationService.cs b/starsky/starsky.feature.thumbnail/Services/DatabaseThumbnailGenerationService.cs index 37e0ff54d1..55a4b7c889 100644 --- a/starsky/starsky.feature.thumbnail/Services/DatabaseThumbnailGenerationService.cs +++ b/starsky/starsky.feature.thumbnail/Services/DatabaseThumbnailGenerationService.cs @@ -45,24 +45,30 @@ public DatabaseThumbnailGenerationService(IQuery query, IWebLogger logger, public async Task StartBackgroundQueue(DateTime endTime) { - var totalProcessed = 0; - var currentPage = 0; - const int batchSize = 100; - if ( _thumbnailQuery.IsRunningJob() ) { return; } - //todo: add a check to see if the service is already running + await _bgTaskQueue.QueueBackgroundWorkItemAsync( + async _ => { await WorkThumbnailGenerationLoop(); }, + "DatabaseThumbnailGenerationService"); + } + + internal async Task WorkThumbnailGenerationLoop() + { + _thumbnailQuery.SetRunningJob(true); List missingThumbnails; + var totalProcessed = 0; + var currentPage = 0; + const int batchSize = 100; do { - // Query missing thumbnails in batches missingThumbnails = - await _thumbnailQuery.GetMissingThumbnailsBatchAsync(currentPage, batchSize); + await _thumbnailQuery.GetMissingThumbnailsBatchAsync(currentPage, + batchSize); // Process each batch var fileHashesList = missingThumbnails.Select(p => p.FileHash).ToList(); @@ -72,16 +78,16 @@ public async Task StartBackgroundQueue(DateTime endTime) break; } - await _bgTaskQueue.QueueBackgroundWorkItemAsync( - async _ => { await WorkThumbnailGeneration(missingThumbnails, queryItems); }, - "DatabaseThumbnailGenerationService"); + await WorkThumbnailGeneration(missingThumbnails, queryItems); totalProcessed += missingThumbnails.Count; currentPage++; _logger.LogInformation( - $"[DatabaseThumbnailGenerationService] Processed {totalProcessed} thumbnails so far..."); + $"[DatabaseThumbnailGenerationService] Processed {totalProcessed} thumbnails so far... ({DateTime.UtcNow:HH:mm:ss})"); } while ( missingThumbnails.Count == batchSize ); + + _thumbnailQuery.SetRunningJob(false); } internal async Task> WorkThumbnailGeneration( @@ -123,7 +129,9 @@ await _updateStatusGeneratedThumbnailService.AddOrUpdateStatusAsync( } var filteredData = resultData - .Where(p => p.Status is FileIndexItem.ExifStatus.Ok or FileIndexItem.ExifStatus.OkAndSame).ToList(); + .Where(p => + p.Status is FileIndexItem.ExifStatus.Ok or FileIndexItem.ExifStatus.OkAndSame) + .ToList(); if ( filteredData.Count == 0 ) { diff --git a/starsky/starsky.foundation.database/Query/QueryGetObjectsByFileHashAsync.cs b/starsky/starsky.foundation.database/Query/QueryGetObjectsByFileHashAsync.cs index 5738883e76..d754236c47 100644 --- a/starsky/starsky.foundation.database/Query/QueryGetObjectsByFileHashAsync.cs +++ b/starsky/starsky.foundation.database/Query/QueryGetObjectsByFileHashAsync.cs @@ -4,50 +4,61 @@ using System.Threading.Tasks; using Microsoft.EntityFrameworkCore; using starsky.foundation.database.Data; -using starsky.foundation.database.Interfaces; using starsky.foundation.database.Models; using starsky.foundation.platform.Helpers; -namespace starsky.foundation.database.Query +namespace starsky.foundation.database.Query; + +/// +/// QueryGetObjectsByFileHashAsync & GetObjectsByFileHashAsync +/// +public partial class Query { - /// - /// GetObjectsByFileHashAsync - /// - public partial class Query : IQuery + public async Task> GetObjectsByFileHashAsync(List fileHashesList, + int retryCount = 2) { - - public async Task> GetObjectsByFileHashAsync(List fileHashesList, int retryCount = 2) + if ( fileHashesList.Count == 0 ) { - if ( fileHashesList.Count == 0 ) - { - return new List(); - } + return new List(); + } - async Task> LocalQuery(ApplicationDbContext context) + async Task> LocalQuery(ApplicationDbContext context) + { + var result = await context.FileIndex.TagWith("GetObjectsByFileHashAsync").Where(p => + fileHashesList.Contains(p.FileHash!)).ToListAsync(); + foreach ( var fileHash in fileHashesList ) { - var result = await context. - FileIndex.TagWith("GetObjectsByFileHashAsync").Where(p => - fileHashesList.Contains(p.FileHash!)).ToListAsync(); - foreach ( var fileHash in fileHashesList ) + if ( result.Find(p => p.FileHash == fileHash) == null ) { - if ( result.Find(p => p.FileHash == fileHash) == null ) + result.Add(new FileIndexItem { - result.Add(new FileIndexItem() - { - FileHash = fileHash, - Status = FileIndexItem.ExifStatus.NotFoundNotInIndex - }); - } + FileHash = fileHash, + Status = FileIndexItem.ExifStatus.NotFoundNotInIndex + }); } - return FormatOk(result); } - async Task> LocalDefaultQuery() + return FormatOk(result); + } + + async Task> LocalDefaultQuery() + { + try { return await LocalQuery(_context); } + // InvalidOperationException can also be disposed + catch ( InvalidOperationException ) + { + if ( _scopeFactory == null ) + { + throw; + } - return await RetryHelper.DoAsync(LocalDefaultQuery, TimeSpan.FromSeconds(3), retryCount); + return await LocalQuery(new InjectServiceScope(_scopeFactory).Context()); + } } + + return await RetryHelper.DoAsync(LocalDefaultQuery, TimeSpan.FromSeconds(3), retryCount); } } diff --git a/starsky/starsky.foundation.database/Thumbnails/ThumbnailQuery.cs b/starsky/starsky.foundation.database/Thumbnails/ThumbnailQuery.cs index c6b4c2b1e1..ef942e99e3 100644 --- a/starsky/starsky.foundation.database/Thumbnails/ThumbnailQuery.cs +++ b/starsky/starsky.foundation.database/Thumbnails/ThumbnailQuery.cs @@ -173,7 +173,29 @@ public bool SetRunningJob(bool value) public async Task> GetMissingThumbnailsBatchAsync(int pageNumber, int pageSize) { - return await _context.Thumbnails + try + { + return await GetMissingThumbnailsBatchInternalAsync(_context, pageNumber, pageSize); + } + // InvalidOperationException can also be disposed + catch ( InvalidOperationException ) + { + if ( _scopeFactory == null ) + { + throw; + } + + return await GetMissingThumbnailsBatchInternalAsync( + new InjectServiceScope(_scopeFactory).Context(), + pageNumber, pageSize); + } + } + + private static async Task> GetMissingThumbnailsBatchInternalAsync( + ApplicationDbContext context, int pageNumber, + int pageSize) + { + return await context.Thumbnails .Where(p => ( p.ExtraLarge == null || p.Large == null || p.Small == null ) && !string.IsNullOrEmpty(p.FileHash)) diff --git a/starsky/starsky.foundation.sync/WatcherBackgroundService/DiskWatcherQueuedHostedService.cs b/starsky/starsky.foundation.sync/WatcherBackgroundService/DiskWatcherQueuedHostedService.cs index 291ed5b744..77d58ebba3 100644 --- a/starsky/starsky.foundation.sync/WatcherBackgroundService/DiskWatcherQueuedHostedService.cs +++ b/starsky/starsky.foundation.sync/WatcherBackgroundService/DiskWatcherQueuedHostedService.cs @@ -8,33 +8,35 @@ using starsky.foundation.worker.Helpers; [assembly: InternalsVisibleTo("starskytest")] -namespace starsky.foundation.sync.WatcherBackgroundService + +namespace starsky.foundation.sync.WatcherBackgroundService; + +[Service(typeof(IHostedService), + InjectionLifetime = InjectionLifetime.Singleton)] +public sealed class DiskWatcherQueuedHostedService : BackgroundService { - [Service(typeof(IHostedService), - InjectionLifetime = InjectionLifetime.Singleton)] - public sealed class DiskWatcherQueuedHostedService : BackgroundService - { - private readonly IDiskWatcherBackgroundTaskQueue _taskQueue; - private readonly IWebLogger _logger; - private readonly AppSettings _appSettings; + private readonly AppSettings _appSettings; + private readonly IWebLogger _logger; + private readonly IDiskWatcherBackgroundTaskQueue _taskQueue; - public DiskWatcherQueuedHostedService( - IDiskWatcherBackgroundTaskQueue taskQueue, - IWebLogger logger, AppSettings appSettings) => - (_taskQueue, _logger, _appSettings) = (taskQueue, logger, appSettings); + public DiskWatcherQueuedHostedService( + IDiskWatcherBackgroundTaskQueue taskQueue, + IWebLogger logger, AppSettings appSettings) + { + ( _taskQueue, _logger, _appSettings ) = ( taskQueue, logger, appSettings ); + } - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - _logger.LogInformation("Queued Hosted Service for DiskWatcher"); - await ProcessTaskQueue.ProcessBatchedLoopAsync(_taskQueue, _logger, - _appSettings, stoppingToken); - } + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Queued Hosted Service for DiskWatcher"); + await ProcessTaskQueue.ProcessBatchedLoopAsync(_taskQueue, _logger, + _appSettings, stoppingToken); + } - public override async Task StopAsync(CancellationToken cancellationToken) - { - _logger.LogInformation( - $"QueuedHostedService {_taskQueue.GetType().Name} is stopping."); - await base.StopAsync(cancellationToken); - } + public override async Task StopAsync(CancellationToken cancellationToken) + { + _logger.LogInformation( + $"QueuedHostedService {_taskQueue.GetType().Name} is stopping. Counts: {_taskQueue.Count()}"); + await base.StopAsync(cancellationToken); } } diff --git a/starsky/starsky.foundation.worker/Services/UpdateBackgroundQueuedHostedService.cs b/starsky/starsky.foundation.worker/Services/UpdateBackgroundQueuedHostedService.cs index fdff0be9f9..c80be12607 100755 --- a/starsky/starsky.foundation.worker/Services/UpdateBackgroundQueuedHostedService.cs +++ b/starsky/starsky.foundation.worker/Services/UpdateBackgroundQueuedHostedService.cs @@ -32,7 +32,7 @@ protected override Task ExecuteAsync(CancellationToken cancellationToken) public override async Task StopAsync(CancellationToken stoppingToken) { _logger.LogInformation( - $"QueuedHostedService {_taskQueue.GetType().Name} is stopping."); + $"QueuedHostedService {_taskQueue.GetType().Name} is stopping. Counts: {_taskQueue.Count()}"); await base.StopAsync(stoppingToken); } } diff --git a/starsky/starsky.foundation.worker/ThumbnailServices/ThumbnailQueuedHostedService.cs b/starsky/starsky.foundation.worker/ThumbnailServices/ThumbnailQueuedHostedService.cs index 2b58d94b46..5a56e7c1d2 100644 --- a/starsky/starsky.foundation.worker/ThumbnailServices/ThumbnailQueuedHostedService.cs +++ b/starsky/starsky.foundation.worker/ThumbnailServices/ThumbnailQueuedHostedService.cs @@ -9,33 +9,34 @@ using starsky.foundation.worker.ThumbnailServices.Interfaces; [assembly: InternalsVisibleTo("starskytest")] -namespace starsky.foundation.worker.ThumbnailServices + +namespace starsky.foundation.worker.ThumbnailServices; + +[Service(typeof(IHostedService), + InjectionLifetime = InjectionLifetime.Singleton)] +public sealed class ThumbnailQueuedHostedService : BackgroundService { - [Service(typeof(IHostedService), - InjectionLifetime = InjectionLifetime.Singleton)] - public sealed class ThumbnailQueuedHostedService : BackgroundService - { - private readonly IThumbnailQueuedHostedService _taskQueue; - private readonly IWebLogger _logger; + private readonly IWebLogger _logger; + private readonly IThumbnailQueuedHostedService _taskQueue; - public ThumbnailQueuedHostedService( - IThumbnailQueuedHostedService taskQueue, - IWebLogger logger, AppSettings appSettings) => - (_taskQueue, _logger, _) = (taskQueue, logger, appSettings); + public ThumbnailQueuedHostedService( + IThumbnailQueuedHostedService taskQueue, + IWebLogger logger, AppSettings appSettings) + { + ( _taskQueue, _logger, _ ) = ( taskQueue, logger, appSettings ); + } - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - _logger.LogInformation("Queued Hosted Service for Thumbnails"); - await ProcessTaskQueue.ProcessTaskQueueAsync(_taskQueue, _logger, - stoppingToken); - } + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + _logger.LogInformation("Queued Hosted Service for Thumbnails"); + await ProcessTaskQueue.ProcessTaskQueueAsync(_taskQueue, _logger, + stoppingToken); + } - public override async Task StopAsync(CancellationToken cancellationToken) - { - _logger.LogInformation( - $"QueuedHostedService {_taskQueue.GetType().Name} is stopping."); - await base.StopAsync(cancellationToken); - } + public override async Task StopAsync(CancellationToken cancellationToken) + { + _logger.LogInformation( + $"QueuedHostedService {_taskQueue.GetType().Name} is stopping. Counts: {_taskQueue.Count()}"); + await base.StopAsync(cancellationToken); } } -