Skip to content

Commit

Permalink
reformat, add counts name && change to add WorkThumbnailGeneration Lo…
Browse files Browse the repository at this point in the history 
…op!. Add Disposed checker
  • Loading branch information
@qdraw
qdraw committed Oct 25, 2024
1 parent 65a3884 commit 0f21c3b
Show file tree
Hide file tree
Showing 6 changed files with 133 additions and 89 deletions.
32 changes: 20 additions & 12 deletions starsky/starsky.feature.thumbnail/Services/DatabaseThumbnailGenerationService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,24 +45,30 @@ public DatabaseThumbnailGenerationService(IQuery query, IWebLogger logger,

public async Task StartBackgroundQueue(DateTime endTime)
{
var totalProcessed = 0;
var currentPage = 0;
const int batchSize = 100;

if ( _thumbnailQuery.IsRunningJob() )
{
return;
}

//todo: add a check to see if the service is already running
await _bgTaskQueue.QueueBackgroundWorkItemAsync(
async _ => { await WorkThumbnailGenerationLoop(); },
"DatabaseThumbnailGenerationService");
}

internal async Task WorkThumbnailGenerationLoop()
{
_thumbnailQuery.SetRunningJob(true);

List<ThumbnailItem> missingThumbnails;
var totalProcessed = 0;
var currentPage = 0;
const int batchSize = 100;

do
{
// Query missing thumbnails in batches
missingThumbnails =
await _thumbnailQuery.GetMissingThumbnailsBatchAsync(currentPage, batchSize);
await _thumbnailQuery.GetMissingThumbnailsBatchAsync(currentPage,
batchSize);

// Process each batch
var fileHashesList = missingThumbnails.Select(p => p.FileHash).ToList();
Expand All @@ -72,16 +78,16 @@ public async Task StartBackgroundQueue(DateTime endTime)
break;
}

await _bgTaskQueue.QueueBackgroundWorkItemAsync(
async _ => { await WorkThumbnailGeneration(missingThumbnails, queryItems); },
"DatabaseThumbnailGenerationService");
await WorkThumbnailGeneration(missingThumbnails, queryItems);

totalProcessed += missingThumbnails.Count;
currentPage++;

_logger.LogInformation(
$"[DatabaseThumbnailGenerationService] Processed {totalProcessed} thumbnails so far...");
$"[DatabaseThumbnailGenerationService] Processed {totalProcessed} thumbnails so far... ({DateTime.UtcNow:HH:mm:ss})");
} while ( missingThumbnails.Count == batchSize );

_thumbnailQuery.SetRunningJob(false);
}

internal async Task<IEnumerable<ThumbnailItem>> WorkThumbnailGeneration(
Expand Down Expand Up @@ -123,7 +129,9 @@ await _updateStatusGeneratedThumbnailService.AddOrUpdateStatusAsync(
}

var filteredData = resultData
.Where(p => p.Status is FileIndexItem.ExifStatus.Ok or FileIndexItem.ExifStatus.OkAndSame).ToList();
.Where(p =>
p.Status is FileIndexItem.ExifStatus.Ok or FileIndexItem.ExifStatus.OkAndSame)
.ToList();

if ( filteredData.Count == 0 )
{
Expand Down
65 changes: 38 additions & 27 deletions starsky/starsky.foundation.database/Query/QueryGetObjectsByFileHashAsync.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,50 +4,61 @@
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using starsky.foundation.database.Data;
using starsky.foundation.database.Interfaces;
using starsky.foundation.database.Models;
using starsky.foundation.platform.Helpers;

namespace starsky.foundation.database.Query
namespace starsky.foundation.database.Query;

/// <summary>
/// QueryGetObjectsByFileHashAsync & GetObjectsByFileHashAsync
/// </summary>
public partial class Query
{
/// <summary>
/// GetObjectsByFileHashAsync
/// </summary>
public partial class Query : IQuery
public async Task<List<FileIndexItem>> GetObjectsByFileHashAsync(List<string> fileHashesList,
int retryCount = 2)
{

public async Task<List<FileIndexItem>> GetObjectsByFileHashAsync(List<string> fileHashesList, int retryCount = 2)
if ( fileHashesList.Count == 0 )
{
if ( fileHashesList.Count == 0 )
{
return new List<FileIndexItem>();
}
return new List<FileIndexItem>();
}

async Task<List<FileIndexItem>> LocalQuery(ApplicationDbContext context)
async Task<List<FileIndexItem>> LocalQuery(ApplicationDbContext context)
{
var result = await context.FileIndex.TagWith("GetObjectsByFileHashAsync").Where(p =>
fileHashesList.Contains(p.FileHash!)).ToListAsync();
foreach ( var fileHash in fileHashesList )
{
var result = await context.
FileIndex.TagWith("GetObjectsByFileHashAsync").Where(p =>
fileHashesList.Contains(p.FileHash!)).ToListAsync();
foreach ( var fileHash in fileHashesList )
if ( result.Find(p => p.FileHash == fileHash) == null )
{
if ( result.Find(p => p.FileHash == fileHash) == null )
result.Add(new FileIndexItem
{
result.Add(new FileIndexItem()
{
FileHash = fileHash,
Status = FileIndexItem.ExifStatus.NotFoundNotInIndex
});
}
FileHash = fileHash,
Status = FileIndexItem.ExifStatus.NotFoundNotInIndex
});
}
return FormatOk(result);
}

async Task<List<FileIndexItem>> LocalDefaultQuery()
return FormatOk(result);
}

async Task<List<FileIndexItem>> LocalDefaultQuery()
{
try
{
return await LocalQuery(_context);
}
// InvalidOperationException can also be disposed
catch ( InvalidOperationException )
{
if ( _scopeFactory == null )
{
throw;
}

return await RetryHelper.DoAsync(LocalDefaultQuery, TimeSpan.FromSeconds(3), retryCount);
return await LocalQuery(new InjectServiceScope(_scopeFactory).Context());
}
}

return await RetryHelper.DoAsync(LocalDefaultQuery, TimeSpan.FromSeconds(3), retryCount);
}
}
24 changes: 23 additions & 1 deletion starsky/starsky.foundation.database/Thumbnails/ThumbnailQuery.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,29 @@ public bool SetRunningJob(bool value)
public async Task<List<ThumbnailItem>> GetMissingThumbnailsBatchAsync(int pageNumber,
int pageSize)
{
return await _context.Thumbnails
try
{
return await GetMissingThumbnailsBatchInternalAsync(_context, pageNumber, pageSize);
}
// InvalidOperationException can also be disposed
catch ( InvalidOperationException )
{
if ( _scopeFactory == null )
{
throw;
}

return await GetMissingThumbnailsBatchInternalAsync(
new InjectServiceScope(_scopeFactory).Context(),
pageNumber, pageSize);
}
}

private static async Task<List<ThumbnailItem>> GetMissingThumbnailsBatchInternalAsync(
ApplicationDbContext context, int pageNumber,
int pageSize)
{
return await context.Thumbnails
.Where(p => ( p.ExtraLarge == null
|| p.Large == null || p.Small == null )
&& !string.IsNullOrEmpty(p.FileHash))
Expand Down
50 changes: 26 additions & 24 deletions starsky/starsky.foundation.sync/WatcherBackgroundService/DiskWatcherQueuedHostedService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,33 +8,35 @@
using starsky.foundation.worker.Helpers;

[assembly: InternalsVisibleTo("starskytest")]
namespace starsky.foundation.sync.WatcherBackgroundService

namespace starsky.foundation.sync.WatcherBackgroundService;

[Service(typeof(IHostedService),
InjectionLifetime = InjectionLifetime.Singleton)]
public sealed class DiskWatcherQueuedHostedService : BackgroundService
{
[Service(typeof(IHostedService),
InjectionLifetime = InjectionLifetime.Singleton)]
public sealed class DiskWatcherQueuedHostedService : BackgroundService
{
private readonly IDiskWatcherBackgroundTaskQueue _taskQueue;
private readonly IWebLogger _logger;
private readonly AppSettings _appSettings;
private readonly AppSettings _appSettings;
private readonly IWebLogger _logger;
private readonly IDiskWatcherBackgroundTaskQueue _taskQueue;

public DiskWatcherQueuedHostedService(
IDiskWatcherBackgroundTaskQueue taskQueue,
IWebLogger logger, AppSettings appSettings) =>
(_taskQueue, _logger, _appSettings) = (taskQueue, logger, appSettings);
public DiskWatcherQueuedHostedService(
IDiskWatcherBackgroundTaskQueue taskQueue,
IWebLogger logger, AppSettings appSettings)
{
( _taskQueue, _logger, _appSettings ) = ( taskQueue, logger, appSettings );
}

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Queued Hosted Service for DiskWatcher");
await ProcessTaskQueue.ProcessBatchedLoopAsync(_taskQueue, _logger,
_appSettings, stoppingToken);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Queued Hosted Service for DiskWatcher");
await ProcessTaskQueue.ProcessBatchedLoopAsync(_taskQueue, _logger,
_appSettings, stoppingToken);
}

public override async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation(
$"QueuedHostedService {_taskQueue.GetType().Name} is stopping.");
await base.StopAsync(cancellationToken);
}
public override async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation(
$"QueuedHostedService {_taskQueue.GetType().Name} is stopping. Counts: {_taskQueue.Count()}");
await base.StopAsync(cancellationToken);
}
}
2 changes: 1 addition & 1 deletion starsky/starsky.foundation.worker/Services/UpdateBackgroundQueuedHostedService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ protected override Task ExecuteAsync(CancellationToken cancellationToken)
public override async Task StopAsync(CancellationToken stoppingToken)
{
_logger.LogInformation(
$"QueuedHostedService {_taskQueue.GetType().Name} is stopping.");
$"QueuedHostedService {_taskQueue.GetType().Name} is stopping. Counts: {_taskQueue.Count()}");
await base.StopAsync(stoppingToken);
}
}
Expand Down
49 changes: 25 additions & 24 deletions starsky/starsky.foundation.worker/ThumbnailServices/ThumbnailQueuedHostedService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,34 @@
using starsky.foundation.worker.ThumbnailServices.Interfaces;

[assembly: InternalsVisibleTo("starskytest")]
namespace starsky.foundation.worker.ThumbnailServices

namespace starsky.foundation.worker.ThumbnailServices;

[Service(typeof(IHostedService),
InjectionLifetime = InjectionLifetime.Singleton)]
public sealed class ThumbnailQueuedHostedService : BackgroundService
{
[Service(typeof(IHostedService),
InjectionLifetime = InjectionLifetime.Singleton)]
public sealed class ThumbnailQueuedHostedService : BackgroundService
{
private readonly IThumbnailQueuedHostedService _taskQueue;
private readonly IWebLogger _logger;
private readonly IWebLogger _logger;
private readonly IThumbnailQueuedHostedService _taskQueue;

public ThumbnailQueuedHostedService(
IThumbnailQueuedHostedService taskQueue,
IWebLogger logger, AppSettings appSettings) =>
(_taskQueue, _logger, _) = (taskQueue, logger, appSettings);
public ThumbnailQueuedHostedService(
IThumbnailQueuedHostedService taskQueue,
IWebLogger logger, AppSettings appSettings)
{
( _taskQueue, _logger, _ ) = ( taskQueue, logger, appSettings );
}

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Queued Hosted Service for Thumbnails");
await ProcessTaskQueue.ProcessTaskQueueAsync(_taskQueue, _logger,
stoppingToken);
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
_logger.LogInformation("Queued Hosted Service for Thumbnails");
await ProcessTaskQueue.ProcessTaskQueueAsync(_taskQueue, _logger,
stoppingToken);
}

public override async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation(
$"QueuedHostedService {_taskQueue.GetType().Name} is stopping.");
await base.StopAsync(cancellationToken);
}
public override async Task StopAsync(CancellationToken cancellationToken)
{
_logger.LogInformation(
$"QueuedHostedService {_taskQueue.GetType().Name} is stopping. Counts: {_taskQueue.Count()}");
await base.StopAsync(cancellationToken);
}
}

0 comments on commit 0f21c3b

Please sign in to comment.