Lots of Bugfixes

API.Tests/Services/ScannerServiceTests.cs

@@ -7,7 +7,10 @@
 using System.Text;
 using System.Text.Json;
 using System.Threading.Tasks;
+using System.Xml;
+using System.Xml.Serialization;
 using API.Data;
+using API.Data.Metadata;
 using API.Data.Repositories;
 using API.Entities;
 using API.Entities.Enums;
@@ -35,6 +38,7 @@ public class ScannerServiceTests : AbstractDbTest
     private readonly string _testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/ScanTests");
     private readonly string _testcasesDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/TestCases");
     private readonly string _imagePath = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/1x1.png");
+    private static readonly string[] ComicInfoExtensions = new[] { ".cbz", ".cbr", ".zip", ".rar" };
 
     public ScannerServiceTests(ITestOutputHelper testOutputHelper)
     {
@@ -125,9 +129,61 @@ public async Task ScanLibrary_FlatSeriesWithSpecial()
         Assert.NotNull(postLib.Series.First().Volumes.FirstOrDefault(v => v.Chapters.FirstOrDefault(c => c.IsSpecial) != null));
     }
 
-    private async Task<Library> GenerateScannerData(string testcase)
+    /// <summary>
+    /// This is testing that if the first file is named A and has a localized name of B if all other files are named B, it should still group and name the series A
+    /// </summary>
+    [Fact]
+    public async Task ScanLibrary_LocalizedSeries()
+    {
+        const string testcase = "Series with Localized - Manga.json";
+
+        // Get the first file and generate a ComicInfo
+        var infos = new Dictionary<string, ComicInfo>();
+        infos.Add("My Dress-Up Darling v01.cbz", new ComicInfo()
+        {
+            Series = "My Dress-Up Darling",
+            LocalizedSeries = "Sono Bisque Doll wa Koi wo Suru"
+        });
+
+        var library = await GenerateScannerData(testcase, infos);
+
+
+        var scanner = CreateServices();
+        await scanner.ScanLibrary(library.Id);
+        var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
+
+        Assert.NotNull(postLib);
+        Assert.Single(postLib.Series);
+        Assert.Equal(3, postLib.Series.First().Volumes.Count);
+    }
+
+
+    /// <summary>
+    /// Files under a folder with a SP marker should group into one issue
+    /// </summary>
+    /// <remarks>https://github.com/Kareadita/Kavita/issues/3299</remarks>
+    [Fact]
+    public async Task ScanLibrary_ImageSeries_SpecialGrouping()
+    {
+        const string testcase = "Image Series with SP Folder - Manga.json";
+
+        var library = await GenerateScannerData(testcase);
+
+
+        var scanner = CreateServices();
+        await scanner.ScanLibrary(library.Id);
+        var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
+
+        Assert.NotNull(postLib);
+        Assert.Single(postLib.Series);
+        Assert.Equal(3, postLib.Series.First().Volumes.Count);
+    }
+
+
+    #region Setup
+    private async Task<Library> GenerateScannerData(string testcase, Dictionary<string, ComicInfo> comicInfos = null)
     {
-        var testDirectoryPath = await GenerateTestDirectory(Path.Join(_testcasesDirectory, testcase));
+        var testDirectoryPath = await GenerateTestDirectory(Path.Join(_testcasesDirectory, testcase), comicInfos);
 
         var (publisher, type) = SplitPublisherAndLibraryType(Path.GetFileNameWithoutExtension(testcase));
 
@@ -148,11 +204,17 @@ private async Task<Library> GenerateScannerData(string testcase)
 
     private ScannerService CreateServices()
     {
-        var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), new FileSystem());
-        var mockReadingService = new MockReadingItemService(ds, Substitute.For<IBookService>());
+        var fs = new FileSystem();
+        var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fs);
+        var archiveService = new ArchiveService(Substitute.For<ILogger<ArchiveService>>(), ds,
+            Substitute.For<IImageService>(), Substitute.For<IMediaErrorService>());
+        var readingItemService = new ReadingItemService(archiveService, Substitute.For<IBookService>(),
+            Substitute.For<IImageService>(), ds, Substitute.For<ILogger<ReadingItemService>>());
+
+
         var processSeries = new ProcessSeries(_unitOfWork, Substitute.For<ILogger<ProcessSeries>>(),
             Substitute.For<IEventHub>(),
-            ds, Substitute.For<ICacheHelper>(), mockReadingService, Substitute.For<IFileService>(),
+            ds, Substitute.For<ICacheHelper>(), readingItemService, new FileService(fs),
             Substitute.For<IMetadataService>(),
             Substitute.For<IWordCountAnalyzerService>(),
             Substitute.For<IReadingListService>(),
@@ -161,7 +223,7 @@ private ScannerService CreateServices()
         var scanner = new ScannerService(_unitOfWork, Substitute.For<ILogger<ScannerService>>(),
             Substitute.For<IMetadataService>(),
             Substitute.For<ICacheService>(), Substitute.For<IEventHub>(), ds,
-            mockReadingService, processSeries, Substitute.For<IWordCountAnalyzerService>());
+            readingItemService, processSeries, Substitute.For<IWordCountAnalyzerService>());
         return scanner;
     }
 
@@ -189,7 +251,7 @@ private static (string Publisher, LibraryType Type) SplitPublisherAndLibraryType
 
 
 
-    private async Task<string> GenerateTestDirectory(string mapPath)
+    private async Task<string> GenerateTestDirectory(string mapPath, Dictionary<string, ComicInfo> comicInfos = null)
     {
         // Read the map file
         var mapContent = await File.ReadAllTextAsync(mapPath);
@@ -206,15 +268,15 @@ private async Task<string> GenerateTestDirectory(string mapPath)
         Directory.CreateDirectory(testDirectory);
 
         // Generate the files and folders
-        await Scaffold(testDirectory, filePaths);
+        await Scaffold(testDirectory, filePaths, comicInfos);
 
         _testOutputHelper.WriteLine($"Test Directory Path: {testDirectory}");
 
         return testDirectory;
     }
 
 
-    private async Task Scaffold(string testDirectory, List<string> filePaths)
+    private async Task Scaffold(string testDirectory, List<string> filePaths, Dictionary<string, ComicInfo> comicInfos = null)
     {
         foreach (var relativePath in filePaths)
         {
@@ -229,9 +291,9 @@ private async Task Scaffold(string testDirectory, List<string> filePaths)
             }
 
             var ext = Path.GetExtension(fullPath).ToLower();
-            if (new[] { ".cbz", ".cbr", ".zip", ".rar" }.Contains(ext))
+            if (ComicInfoExtensions.Contains(ext) && comicInfos != null && comicInfos.TryGetValue(Path.GetFileName(relativePath), out var info))
             {
-                CreateMinimalCbz(fullPath, includeMetadata: true);
+                CreateMinimalCbz(fullPath, info);
             }
             else
             {
@@ -242,54 +304,44 @@ private async Task Scaffold(string testDirectory, List<string> filePaths)
         }
     }
 
-    private void CreateMinimalCbz(string filePath, bool includeMetadata)
+    private void CreateMinimalCbz(string filePath, ComicInfo? comicInfo = null)
     {
-        var tempImagePath = _imagePath; // Assuming _imagePath is a valid path to the 1x1 image
-
         using (var archive = ZipFile.Open(filePath, ZipArchiveMode.Create))
         {
             // Add the 1x1 image to the archive
-            archive.CreateEntryFromFile(tempImagePath, "1x1.png");
+            archive.CreateEntryFromFile(_imagePath, "1x1.png");
 
-            if (includeMetadata)
+            if (comicInfo != null)
             {
-                var comicInfo = GenerateComicInfo();
+                // Serialize ComicInfo object to XML
+                var comicInfoXml = SerializeComicInfoToXml(comicInfo);
+
+                // Create an entry for ComicInfo.xml in the archive
                 var entry = archive.CreateEntry("ComicInfo.xml");
                 using var entryStream = entry.Open();
                 using var writer = new StreamWriter(entryStream, Encoding.UTF8);
-                writer.Write(comicInfo);
+
+                // Write the XML to the archive
+                writer.Write(comicInfoXml);
             }
+
         }
-        Console.WriteLine($"Created minimal CBZ archive: {filePath} with{(includeMetadata ? "" : "out")} metadata.");
+        Console.WriteLine($"Created minimal CBZ archive: {filePath} with{(comicInfo != null ? "" : "out")} metadata.");
     }
 
-    private string GenerateComicInfo()
-    {
-        var comicInfo = new StringBuilder();
-        comicInfo.AppendLine("<?xml version='1.0' encoding='utf-8'?>");
-        comicInfo.AppendLine("<ComicInfo>");
-
-        // People Tags
-        string[] people = { "Joe Shmo", "Tommy Two Hands"};
-        string[] genres = { /* Your list of genres here */ };
-
-        void AddRandomTag(string tagName, string[] choices)
-        {
-            if (new Random().Next(0, 2) == 1) // 50% chance to include the tag
-            {
-                var selected = choices.OrderBy(x => Guid.NewGuid()).Take(new Random().Next(1, 5)).ToArray();
-                comicInfo.AppendLine($"  <{tagName}>{string.Join(", ", selected)}</{tagName}>");
-            }
-        }
 
-        foreach (var tag in new[] { "Writer", "Penciller", "Inker", "CoverArtist", "Publisher", "Character", "Imprint", "Colorist", "Letterer", "Editor", "Translator", "Team", "Location" })
+    private static string SerializeComicInfoToXml(ComicInfo comicInfo)
+    {
+        var xmlSerializer = new XmlSerializer(typeof(ComicInfo));
+        using var stringWriter = new StringWriter();
+        using (var xmlWriter = XmlWriter.Create(stringWriter, new XmlWriterSettings { Indent = true, Encoding = new UTF8Encoding(false), OmitXmlDeclaration = false}))
         {
-            AddRandomTag(tag, people);
+            xmlSerializer.Serialize(xmlWriter, comicInfo);
         }
 
-        AddRandomTag("Genre", genres);
-        comicInfo.AppendLine("</ComicInfo>");
-
-        return comicInfo.ToString();
+        // For the love of god, I spent 2 hours trying to get utf-8 with no BOM
+        return stringWriter.ToString().Replace("""<?xml version="1.0" encoding="utf-16"?>""",
+            @"<?xml version='1.0' encoding='utf-8'?>");
     }
+    #endregion
 }

API.Tests/Services/Test Data/ScannerService/TestCases/Image Series with SP Folder - Manga.json

@@ -0,0 +1,6 @@
+[
+    "My Dress-Up Darling/My Dress-Up Darling vol 1/0001.png",
+    "My Dress-Up Darling/My Dress-Up Darling vol 1/0002.png",
+    "My Dress-Up Darling/My Dress-Up Darling vol 2/0001.png",
+    "My Dress-Up Darling/Specials/My Dress-Up Darling SP01/0001.png"
+]

API.Tests/Services/Test Data/ScannerService/TestCases/Series with Localized - Manga.json

@@ -0,0 +1,5 @@
+[
+    "My Dress-Up Darling/My Dress-Up Darling v01.cbz",
+    "My Dress-Up Darling/Sono Bisque Doll wa Koi wo Suru v02.cbz",
+    "My Dress-Up Darling/Sono Bisque Doll wa Koi wo Suru ch 10.cbz"
+]

API/Controllers/PersonController.cs

@@ -11,6 +11,7 @@
 using Nager.ArticleNumber;
 
 namespace API.Controllers;
+#nullable enable
 
 public class PersonController : BaseApiController
 {
@@ -39,11 +40,11 @@ public async Task<ActionResult<IEnumerable<PersonRole>>> GetRolesForPersonByName
     }
 
     /// <summary>
-    /// Returns a list of authors for browsing
+    /// Returns a list of authors & artists for browsing
     /// </summary>
     /// <param name="userParams"></param>
     /// <returns></returns>
-    [HttpPost("authors")]
+    [HttpPost("all")]
     public async Task<ActionResult<PagedList<BrowsePersonDto>>> GetAuthorsForBrowse([FromQuery] UserParams? userParams)
     {
         userParams ??= UserParams.Default;

API/Data/Repositories/PersonRepository.cs

@@ -171,6 +171,7 @@ public async Task<PagedList<BrowsePersonDto>> GetAllWritersAndSeriesCount(int us
                 Id = p.Id,
                 Name = p.Name,
                 Description = p.Description,
+                CoverImage = p.CoverImage,
                 SeriesCount = p.SeriesMetadataPeople
                     .Where(smp => roles.Contains(smp.Role))
                     .Select(smp => smp.SeriesMetadata.SeriesId)

API/Extensions/QueryExtensions/Filtering/SeriesFilter.cs

@@ -424,7 +424,7 @@ public static IQueryable<Series> HasReadingDate(this IQueryable<Series> queryabl
     public static IQueryable<Series> HasTags(this IQueryable<Series> queryable, bool condition,
         FilterComparison comparison, IList<int> tags)
     {
-        if (!condition || tags.Count == 0) return queryable;
+        if (!condition || (comparison != FilterComparison.IsEmpty && tags.Count == 0)) return queryable;
 
         switch (comparison)
         {
@@ -547,7 +547,7 @@ public static IQueryable<Series> HasPeopleLegacy(this IQueryable<Series> queryab
     public static IQueryable<Series> HasGenre(this IQueryable<Series> queryable, bool condition,
         FilterComparison comparison, IList<int> genres)
     {
-        if (!condition || genres.Count == 0) return queryable;
+        if (!condition || (comparison != FilterComparison.IsEmpty && genres.Count == 0)) return queryable;
 
         switch (comparison)
         {
@@ -620,7 +620,7 @@ public static IQueryable<Series> HasFormat(this IQueryable<Series> queryable, bo
     public static IQueryable<Series> HasCollectionTags(this IQueryable<Series> queryable, bool condition,
         FilterComparison comparison, IList<int> collectionTags, IList<int> collectionSeries)
     {
-        if (!condition || collectionTags.Count == 0) return queryable;
+        if (!condition || (comparison != FilterComparison.IsEmpty && collectionTags.Count == 0)) return queryable;
 
 
         switch (comparison)

API/Services/ReadingItemService.cs

@@ -9,7 +9,6 @@ namespace API.Services;
 
 public interface IReadingItemService
 {
-    ComicInfo? GetComicInfo(string filePath);
     int GetNumberOfPages(string filePath, MangaFormat format);
     string GetCoverImage(string filePath, string fileName, MangaFormat format, EncodeFormat encodeFormat, CoverImageSize size = CoverImageSize.Default);
     void Extract(string fileFilePath, string targetDirectory, MangaFormat format, int imageCount = 1);
@@ -51,7 +50,7 @@ public ReadingItemService(IArchiveService archiveService, IBookService bookServi
     /// </summary>
     /// <param name="filePath">Fully qualified path of file</param>
     /// <returns></returns>
-    public ComicInfo? GetComicInfo(string filePath)
+    private ComicInfo? GetComicInfo(string filePath)
     {
         if (Parser.IsEpub(filePath))
         {

API/Services/Tasks/Scanner/ProcessSeries.cs

@@ -110,7 +110,7 @@ await _unitOfWork.SeriesRepository.GetFullSeriesByAnyName(firstInfo.Series, firs
 
         try
         {
-            _logger.LogInformation("[ScannerService] Processing series {SeriesName}", series.OriginalName);
+            _logger.LogInformation("[ScannerService] Processing series {SeriesName} with {Count} files", series.OriginalName, parsedInfos.Count);
 
             // parsedInfos[0] is not the first volume or chapter. We need to find it using a ComicInfo check (as it uses firstParsedInfo for series sort)
             var firstParsedInfo = parsedInfos.FirstOrDefault(p => p.ComicInfo != null, firstInfo);
@@ -423,7 +423,7 @@ private async Task UpdateCollectionTags(Series series, Chapter firstChapter)
         var defaultAdmin = await _unitOfWork.UserRepository.GetDefaultAdminUser(AppUserIncludes.Collections);
         if (defaultAdmin == null) return;
 
-        _logger.LogDebug("Collection tag(s) found for {SeriesName}, updating collections", series.Name);
+        _logger.LogInformation("Collection tag(s) found for {SeriesName}, updating collections", series.Name);
         var sw = Stopwatch.StartNew();
 
         foreach (var collection in firstChapter.SeriesGroup.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
@@ -593,7 +593,6 @@ private async Task UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, b
     {
         // Add new volumes and update chapters per volume
         var distinctVolumes = parsedInfos.DistinctVolumes();
-        _logger.LogTrace("[ScannerService] Updating {DistinctVolumes} volumes on {SeriesName}", distinctVolumes.Count, series.Name);
         foreach (var volumeNumber in distinctVolumes)
         {
             Volume? volume;
@@ -621,7 +620,6 @@ private async Task UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, b
             volume.LookupName = volumeNumber;
             volume.Name = volume.GetNumberTitle();
 
-            _logger.LogTrace("[ScannerService] Parsing {SeriesName} - Volume {VolumeNumber}", series.Name, volume.Name);
             var infos = parsedInfos.Where(p => p.Volumes == volumeNumber).ToArray();
 
             await UpdateChapters(series, volume, infos, forceUpdate);
@@ -641,7 +639,7 @@ private void RemoveVolumes(Series series, IList<ParserInfo> parsedInfos)
         if (series.Volumes.Count == nonDeletedVolumes.Count) return;
 
 
-        _logger.LogTrace("[ScannerService] Removed {Count} volumes from {SeriesName} where parsed infos were not mapping with volume name",
+        _logger.LogDebug("[ScannerService] Removed {Count} volumes from {SeriesName} where parsed infos were not mapping with volume name",
             (series.Volumes.Count - nonDeletedVolumes.Count), series.Name);
         var deletedVolumes = series.Volumes.Except(nonDeletedVolumes);
         foreach (var volume in deletedVolumes)
@@ -655,7 +653,7 @@ private void RemoveVolumes(Series series, IList<ParserInfo> parsedInfos)
                     file);
             }
 
-            _logger.LogTrace("[ScannerService] Removed {SeriesName} - Volume {Volume}: {File}", series.Name, volume.Name, file);
+            _logger.LogDebug("[ScannerService] Removed {SeriesName} - Volume {Volume}: {File}", series.Name, volume.Name, file);
         }
 
         series.Volumes = nonDeletedVolumes;
@@ -681,7 +679,7 @@ private async Task UpdateChapters(Series series, Volume volume, IList<ParserInfo
 
             if (chapter == null)
             {
-                _logger.LogTrace(
+                _logger.LogDebug(
                     "[ScannerService] Adding new chapter, {Series} - Vol {Volume} Ch {Chapter}", info.Series, info.Volumes, info.Chapters);
                 chapter = ChapterBuilder.FromParserInfo(info).Build();
                 volume.Chapters.Add(chapter);
@@ -778,7 +776,7 @@ private void RemoveChapters(Volume volume, IList<ParserInfo> parsedInfos)
                 // If no files remain after filtering, remove the chapter
                 if (existingChapter.Files.Count != 0) continue;
 
-                _logger.LogTrace("[ScannerService] Removed chapter {Chapter} for Volume {VolumeNumber} on {SeriesName}",
+                _logger.LogDebug("[ScannerService] Removed chapter {Chapter} for Volume {VolumeNumber} on {SeriesName}",
                     existingChapter.Range, volume.Name, parsedInfos[0].Series);
                 volume.Chapters.Remove(existingChapter);
             }
@@ -789,7 +787,7 @@ private void RemoveChapters(Volume volume, IList<ParserInfo> parsedInfos)
 
                 // If no files exist, remove the chapter
                 if (filesExist) continue;
-                _logger.LogTrace("[ScannerService] Removed chapter {Chapter} for Volume {VolumeNumber} on {SeriesName} as no files exist",
+                _logger.LogDebug("[ScannerService] Removed chapter {Chapter} for Volume {VolumeNumber} on {SeriesName} as no files exist",
                     existingChapter.Range, volume.Name, parsedInfos[0].Series);
                 volume.Chapters.Remove(existingChapter);
             }