Scan Loop Fixes (#1459)

* Added Last Folder Scanned time to series info modal.

Tweaked the info event detail modal to have a primary and thus be auto-dismissable

* Added an error event when multiple series are found in processing a series.

* Fixed a bug where a series could get stuck with other series due to a bad select query.

Started adding the force flag hook for the UI and designing the confirm.

Confirm service now also has ability to hide the close button.

Updated error events and logging in the loop, to be more informative

* Fixed a bug where confirm service wasn't showing the proper body content.

* Hooked up force scan series

* refresh metadata now has force update

* Fixed up the messaging with the prompt on scan, hooked it up properly in the scan library to avoid the check if the whole library needs to even be scanned. Fixed a bug where NormalizedLocalizedName wasn't being calculated on new entities.

Started adding unit tests for this problematic repo method.

* Fixed a bug where we updated NormalizedLocalizedName before we set it.

* Send an info to the UI when series are spread between multiple library level folders.

* Added some logger output when there are no files found in a folder. Return early if there are no files found, so we can avoid some small loops of code.

* Fixed an issue where multiple series in a folder with localized series would cause unintended grouping. This is not supported and hence we will warn them and allow the bad grouping.

* Added a case where scan series fails due to the folder being removed. We will now log an error

* Normalize paths when finding the highest directory till root.

* Fixed an issue with Scan Series where changing a series' folder to a different path but the original series folder existed with another series in it, would cause the series to not be deleted.

* Fixed some bugs around specials causing a series merge issue on scan series.

* Removed a bug marker

* Cleaned up some of the scan loop and removed a test I don't need.

* Remove any prompts for force flow, it doesn't work well. Leave the API as is though.

* Fixed up a check for duplicate ScanLibrary calls

API.Tests/Repository/SeriesRepositoryTests.cs

@@ -0,0 +1,156 @@
+using System.Collections.Generic;
+using System.Data.Common;
+using System.IO.Abstractions.TestingHelpers;
+using System.Linq;
+using System.Threading.Tasks;
+using API.Data;
+using API.Entities;
+using API.Entities.Enums;
+using API.Helpers;
+using API.Services;
+using AutoMapper;
+using Microsoft.Data.Sqlite;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.Extensions.Logging;
+using NSubstitute;
+using Xunit;
+
+namespace API.Tests.Repository;
+
+public class SeriesRepositoryTests
+{
+    private readonly IUnitOfWork _unitOfWork;
+
+    private readonly DbConnection _connection;
+    private readonly DataContext _context;
+
+    private const string CacheDirectory = "C:/kavita/config/cache/";
+    private const string CoverImageDirectory = "C:/kavita/config/covers/";
+    private const string BackupDirectory = "C:/kavita/config/backups/";
+    private const string DataDirectory = "C:/data/";
+
+    public SeriesRepositoryTests()
+    {
+        var contextOptions = new DbContextOptionsBuilder().UseSqlite(CreateInMemoryDatabase()).Options;
+        _connection = RelationalOptionsExtension.Extract(contextOptions).Connection;
+
+        _context = new DataContext(contextOptions);
+        Task.Run(SeedDb).GetAwaiter().GetResult();
+
+        var config = new MapperConfiguration(cfg => cfg.AddProfile<AutoMapperProfiles>());
+        var mapper = config.CreateMapper();
+        _unitOfWork = new UnitOfWork(_context, mapper, null);
+    }
+
+     #region Setup
+
+    private static DbConnection CreateInMemoryDatabase()
+    {
+        var connection = new SqliteConnection("Filename=:memory:");
+
+        connection.Open();
+
+        return connection;
+    }
+
+    private async Task<bool> SeedDb()
+    {
+        await _context.Database.MigrateAsync();
+        var filesystem = CreateFileSystem();
+
+        await Seed.SeedSettings(_context,
+            new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), filesystem));
+
+        var setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.CacheDirectory).SingleAsync();
+        setting.Value = CacheDirectory;
+
+        setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.BackupDirectory).SingleAsync();
+        setting.Value = BackupDirectory;
+
+        _context.ServerSetting.Update(setting);
+
+        var lib = new Library()
+        {
+            Name = "Manga", Folders = new List<FolderPath>() {new FolderPath() {Path = "C:/data/"}}
+        };
+
+        _context.AppUser.Add(new AppUser()
+        {
+            UserName = "majora2007",
+            Libraries = new List<Library>()
+            {
+                lib
+            }
+        });
+
+        return await _context.SaveChangesAsync() > 0;
+    }
+
+    private async Task ResetDb()
+    {
+        _context.Series.RemoveRange(_context.Series.ToList());
+        _context.AppUserRating.RemoveRange(_context.AppUserRating.ToList());
+        _context.Genre.RemoveRange(_context.Genre.ToList());
+        _context.CollectionTag.RemoveRange(_context.CollectionTag.ToList());
+        _context.Person.RemoveRange(_context.Person.ToList());
+
+        await _context.SaveChangesAsync();
+    }
+
+    private static MockFileSystem CreateFileSystem()
+    {
+        var fileSystem = new MockFileSystem();
+        fileSystem.Directory.SetCurrentDirectory("C:/kavita/");
+        fileSystem.AddDirectory("C:/kavita/config/");
+        fileSystem.AddDirectory(CacheDirectory);
+        fileSystem.AddDirectory(CoverImageDirectory);
+        fileSystem.AddDirectory(BackupDirectory);
+        fileSystem.AddDirectory(DataDirectory);
+
+        return fileSystem;
+    }
+
+    #endregion
+
+    private async Task SetupSeriesData()
+    {
+        var library = new Library()
+        {
+            Name = "Manga",
+            Type = LibraryType.Manga,
+            Folders = new List<FolderPath>()
+            {
+                new FolderPath() {Path = "C:/data/manga/"}
+            }
+        };
+
+        library.Series = new List<Series>()
+        {
+            DbFactory.Series("The Idaten Deities Know Only Peace", "Heion Sedai no Idaten-tachi"),
+        };
+
+        _unitOfWork.LibraryRepository.Add(library);
+        await _unitOfWork.CommitAsync();
+    }
+
+
+    [InlineData("Heion Sedai no Idaten-tachi", "", "The Idaten Deities Know Only Peace")] // Matching on localized name in DB
+    public async Task GetFullSeriesByAnyName_Should(string seriesName, string localizedName, string? expected)
+    {
+        var firstSeries = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(1);
+        var series =
+            await _unitOfWork.SeriesRepository.GetFullSeriesByAnyName(seriesName, localizedName,
+                1);
+        if (expected == null)
+        {
+            Assert.Null(series);
+        }
+        else
+        {
+            Assert.NotNull(series);
+            Assert.Equal(expected, series.Name);
+        }
+    }
+
+}

API.Tests/Services/ParseScannedFilesTests.cs

@@ -156,96 +156,6 @@ private static MockFileSystem CreateFileSystem()
 
     #endregion
 
-    #region GetInfosByName
-
-    [Fact]
-    public void GetInfosByName_ShouldReturnGivenMatchingSeriesName()
-    {
-        var fileSystem = new MockFileSystem();
-        var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
-        var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
-            new MockReadingItemService(new DefaultParser(ds)), Substitute.For<IEventHub>());
-
-        var infos = new List<ParserInfo>()
-        {
-            ParserInfoFactory.CreateParsedInfo("Accel World", "1", "0", "Accel World v1.cbz", false),
-            ParserInfoFactory.CreateParsedInfo("Accel World", "2", "0", "Accel World v2.cbz", false)
-        };
-        var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>
-        {
-            {
-                new ParsedSeries()
-                {
-                    Format = MangaFormat.Archive,
-                    Name = "Accel World",
-                    NormalizedName = API.Parser.Parser.Normalize("Accel World")
-                },
-                infos
-            },
-            {
-                new ParsedSeries()
-                {
-                    Format = MangaFormat.Pdf,
-                    Name = "Accel World",
-                    NormalizedName = API.Parser.Parser.Normalize("Accel World")
-                },
-                new List<ParserInfo>()
-            }
-        };
-
-        var series = DbFactory.Series("Accel World");
-        series.Format = MangaFormat.Pdf;
-
-        Assert.Empty(ParseScannedFiles.GetInfosByName(parsedSeries, series));
-
-        series.Format = MangaFormat.Archive;
-        Assert.Equal(2, ParseScannedFiles.GetInfosByName(parsedSeries, series).Count());
-
-    }
-
-    [Fact]
-    public void GetInfosByName_ShouldReturnGivenMatchingNormalizedSeriesName()
-    {
-        var fileSystem = new MockFileSystem();
-        var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
-        var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
-            new MockReadingItemService(new DefaultParser(ds)), Substitute.For<IEventHub>());
-
-        var infos = new List<ParserInfo>()
-        {
-            ParserInfoFactory.CreateParsedInfo("Accel World", "1", "0", "Accel World v1.cbz", false),
-            ParserInfoFactory.CreateParsedInfo("Accel World", "2", "0", "Accel World v2.cbz", false)
-        };
-        var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>
-        {
-            {
-                new ParsedSeries()
-                {
-                    Format = MangaFormat.Archive,
-                    Name = "Accel World",
-                    NormalizedName = API.Parser.Parser.Normalize("Accel World")
-                },
-                infos
-            },
-            {
-                new ParsedSeries()
-                {
-                    Format = MangaFormat.Pdf,
-                    Name = "Accel World",
-                    NormalizedName = API.Parser.Parser.Normalize("Accel World")
-                },
-                new List<ParserInfo>()
-            }
-        };
-
-        var series = DbFactory.Series("accel world");
-        series.Format = MangaFormat.Archive;
-        Assert.Equal(2, ParseScannedFiles.GetInfosByName(parsedSeries, series).Count());
-
-    }
-
-    #endregion
-
     #region MergeName
 
     // NOTE: I don't think I can test MergeName as it relies on Tracking Files, which is more complicated than I need

API/Controllers/LibraryController.cs

@@ -168,17 +168,17 @@ public async Task<ActionResult<MemberDto>> UpdateUserLibraries(UpdateLibraryForU
 
         [Authorize(Policy = "RequireAdminRole")]
         [HttpPost("scan")]
-        public ActionResult Scan(int libraryId)
+        public ActionResult Scan(int libraryId, bool force = false)
         {
-            _taskScheduler.ScanLibrary(libraryId);
+            _taskScheduler.ScanLibrary(libraryId, force);
             return Ok();
         }
 
         [Authorize(Policy = "RequireAdminRole")]
         [HttpPost("refresh-metadata")]
-        public ActionResult RefreshMetadata(int libraryId)
+        public ActionResult RefreshMetadata(int libraryId, bool force = true)
         {
-            _taskScheduler.RefreshMetadata(libraryId);
+            _taskScheduler.RefreshMetadata(libraryId, force);
             return Ok();
         }
 

API/DTOs/SeriesDto.cs

@@ -58,5 +58,9 @@ public class SeriesDto : IHasReadTimeEstimate
         /// The highest level folder for this Series
         /// </summary>
         public string FolderPath { get; set; }
+        /// <summary>
+        /// The last time the folder for this series was scanned
+        /// </summary>
+        public DateTime LastFolderScanned { get; set; }
     }
 }

API/Data/DbFactory.cs

@@ -24,6 +24,26 @@ public static Series Series(string name)
                 OriginalName = name,
                 LocalizedName = name,
                 NormalizedName = Parser.Parser.Normalize(name),
+                NormalizedLocalizedName = Parser.Parser.Normalize(name),
+                SortName = name,
+                Volumes = new List<Volume>(),
+                Metadata = SeriesMetadata(Array.Empty<CollectionTag>())
+            };
+        }
+
+        public static Series Series(string name, string localizedName)
+        {
+            if (string.IsNullOrEmpty(localizedName))
+            {
+                localizedName = name;
+            }
+            return new Series
+            {
+                Name = name,
+                OriginalName = name,
+                LocalizedName = localizedName,
+                NormalizedName = Parser.Parser.Normalize(name),
+                NormalizedLocalizedName = Parser.Parser.Normalize(localizedName),
                 SortName = name,
                 Volumes = new List<Volume>(),
                 Metadata = SeriesMetadata(Array.Empty<CollectionTag>())

API/Data/Repositories/SeriesRepository.cs

@@ -1220,15 +1220,19 @@ public Task<Series> GetFullSeriesByName(string series, int libraryId)
     /// <returns></returns>
     public Task<Series> GetFullSeriesByAnyName(string seriesName, string localizedName, int libraryId)
     {
-        var localizedSeries = Parser.Parser.Normalize(seriesName);
+        var normalizedSeries = Parser.Parser.Normalize(seriesName);
         var normalizedLocalized = Parser.Parser.Normalize(localizedName);
-        return _context.Series
-            .Where(s => s.NormalizedName.Equals(localizedSeries)
-                        || s.NormalizedName.Equals(normalizedLocalized)
-                        || s.NormalizedLocalizedName.Equals(localizedSeries)
-                        || s.NormalizedLocalizedName.Equals(normalizedLocalized))
+        var query = _context.Series
             .Where(s => s.LibraryId == libraryId)
-            .Include(s => s.Metadata)
+            .Where(s => s.NormalizedName.Equals(normalizedSeries)
+                        || (s.NormalizedLocalizedName.Equals(normalizedSeries) && s.NormalizedLocalizedName != string.Empty));
+        if (!string.IsNullOrEmpty(normalizedLocalized))
+        {
+            query = query.Where(s =>
+                s.NormalizedName.Equals(normalizedLocalized) || s.NormalizedLocalizedName.Equals(normalizedLocalized));
+        }
+
+        return query.Include(s => s.Metadata)
             .ThenInclude(m => m.People)
             .Include(s => s.Metadata)
             .ThenInclude(m => m.Genres)

API/Services/DirectoryService.cs

@@ -492,10 +492,10 @@ public Dictionary<string, string> FindHighestDirectoriesFromFiles(IEnumerable<st
        {
            var stopLookingForDirectories = false;
            var dirs = new Dictionary<string, string>();
-           foreach (var folder in libraryFolders)
+           foreach (var folder in libraryFolders.Select(Parser.Parser.NormalizePath))
            {
                if (stopLookingForDirectories) break;
-               foreach (var file in filePaths)
+               foreach (var file in filePaths.Select(Parser.Parser.NormalizePath))
                {
                    if (!file.Contains(folder)) continue;