From 7cb547f2f9420fa9842b9fb1cf174f102fcfc32d Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Sat, 20 Aug 2022 11:35:31 -0500 Subject: [PATCH] New Scan Loop Fixes (#1452) * Refactored ScanSeries to avoid a lot of extra work and fixed a bug where Scan Series would invoke the processing twice. Refactored the series selection code during process such that we use Localized Name as well, for cases where the original name was changed. Undid an optimization around Last Write time, since Linux file systems match how NTFS works. * Fixed part of the query * Added a NormalizedLocalizedName for quick searching in which a series needs grouping. Reworked scan loop code a bit to ensure we don't do extra work. Tweaked the widget logic to help display better and not show "Nothing going on here". * Fixed a bug where archives with ._ files would be counted as valid files, while they are actually just metadata files on Mac's. * Fixed a broken unit test --- API.Tests/Services/ArchiveServiceTests.cs | 1 + .../Archives/macos_withdotunder_one.zip | Bin 0 -> 424 bytes API/Controllers/SeriesController.cs | 2 + API/Data/MigrateNormalizedLocalizedName.cs | 38 + ...223212_NormalizedLocalizedName.Designer.cs | 1608 +++++++++++++++++ .../20220819223212_NormalizedLocalizedName.cs | 25 + .../Migrations/DataContextModelSnapshot.cs | 3 + API/Data/Repositories/SeriesRepository.cs | 58 +- API/Entities/Series.cs | 4 + API/Parser/Parser.cs | 8 +- API/Services/DirectoryService.cs | 6 - API/Services/SeriesService.cs | 1 - API/Services/Tasks/Scanner/ProcessSeries.cs | 26 +- API/Services/Tasks/ScannerService.cs | 91 +- API/Startup.cs | 4 + UI/Web/src/app/_services/account.service.ts | 3 +- .../events-widget.component.html | 2 +- 17 files changed, 1815 insertions(+), 65 deletions(-) create mode 100644 API.Tests/Services/Test Data/ArchiveService/Archives/macos_withdotunder_one.zip create mode 100644 API/Data/MigrateNormalizedLocalizedName.cs create mode 100644 API/Data/Migrations/20220819223212_NormalizedLocalizedName.Designer.cs create mode 100644 API/Data/Migrations/20220819223212_NormalizedLocalizedName.cs diff --git a/API.Tests/Services/ArchiveServiceTests.cs b/API.Tests/Services/ArchiveServiceTests.cs index de27464c9b..5e12a1aeec 100644 --- a/API.Tests/Services/ArchiveServiceTests.cs +++ b/API.Tests/Services/ArchiveServiceTests.cs @@ -68,6 +68,7 @@ public void IsValidArchiveTest(string archivePath, bool expected) [InlineData("macos_none.zip", 0)] [InlineData("macos_one.zip", 1)] [InlineData("macos_native.zip", 21)] + [InlineData("macos_withdotunder_one.zip", 1)] public void GetNumberOfPagesFromArchiveTest(string archivePath, int expected) { var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService/Archives"); diff --git a/API.Tests/Services/Test Data/ArchiveService/Archives/macos_withdotunder_one.zip b/API.Tests/Services/Test Data/ArchiveService/Archives/macos_withdotunder_one.zip new file mode 100644 index 0000000000000000000000000000000000000000..00ce4eb53f77bc837ea1cc185af93864af05a341 GIT binary patch literal 424 zcmWIWW@h1H0D(shEkR%glwbwYC8@x?jZ=gGnX5I?{hVJxPDVa9^Y zQ3hhLu`m}Q8-IrHP6^03bQ;-sOiv;k?*}s;;bmkC7F9Bafh +/// v0.5.6 introduced Normalized Localized Name, which allows for faster lookups and less memory usage. This migration will calculate them once +/// +public static class MigrateNormalizedLocalizedName +{ + public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger logger) + { + if (!await dataContext.Series.Where(s => s.NormalizedLocalizedName == null).AnyAsync()) + { + return; + } + logger.LogInformation("Running MigrateNormalizedLocalizedName migration. Please be patient, this may take some time"); + + + foreach (var series in await dataContext.Series.ToListAsync()) + { + series.NormalizedLocalizedName = Parser.Parser.Normalize(series.LocalizedName ?? string.Empty); + logger.LogInformation("Updated {SeriesName} normalized localized name: {LocalizedName}", series.Name, series.NormalizedLocalizedName); + unitOfWork.SeriesRepository.Update(series); + } + + if (unitOfWork.HasChanges()) + { + await unitOfWork.CommitAsync(); + } + + logger.LogInformation("MigrateNormalizedLocalizedName migration finished"); + + } + +} diff --git a/API/Data/Migrations/20220819223212_NormalizedLocalizedName.Designer.cs b/API/Data/Migrations/20220819223212_NormalizedLocalizedName.Designer.cs new file mode 100644 index 0000000000..41bf29e94a --- /dev/null +++ b/API/Data/Migrations/20220819223212_NormalizedLocalizedName.Designer.cs @@ -0,0 +1,1608 @@ +// +using System; +using API.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace API.Data.Migrations +{ + [DbContext(typeof(DataContext))] + [Migration("20220819223212_NormalizedLocalizedName")] + partial class NormalizedLocalizedName + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "6.0.7"); + + modelBuilder.Entity("API.Entities.AppRole", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("TEXT"); + + b.Property("Name") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("NormalizedName") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique() + .HasDatabaseName("RoleNameIndex"); + + b.ToTable("AspNetRoles", (string)null); + }); + + modelBuilder.Entity("API.Entities.AppUser", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AccessFailedCount") + .HasColumnType("INTEGER"); + + b.Property("ApiKey") + .HasColumnType("TEXT"); + + b.Property("ConcurrencyStamp") + .IsConcurrencyToken() + .HasColumnType("TEXT"); + + b.Property("ConfirmationToken") + .HasColumnType("TEXT"); + + b.Property("Created") + .HasColumnType("TEXT"); + + b.Property("Email") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("EmailConfirmed") + .HasColumnType("INTEGER"); + + b.Property("LastActive") + .HasColumnType("TEXT"); + + b.Property("LockoutEnabled") + .HasColumnType("INTEGER"); + + b.Property("LockoutEnd") + .HasColumnType("TEXT"); + + b.Property("NormalizedEmail") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("NormalizedUserName") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("PasswordHash") + .HasColumnType("TEXT"); + + b.Property("PhoneNumber") + .HasColumnType("TEXT"); + + b.Property("PhoneNumberConfirmed") + .HasColumnType("INTEGER"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .HasColumnType("INTEGER"); + + b.Property("SecurityStamp") + .HasColumnType("TEXT"); + + b.Property("TwoFactorEnabled") + .HasColumnType("INTEGER"); + + b.Property("UserName") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedEmail") + .HasDatabaseName("EmailIndex"); + + b.HasIndex("NormalizedUserName") + .IsUnique() + .HasDatabaseName("UserNameIndex"); + + b.ToTable("AspNetUsers", (string)null); + }); + + modelBuilder.Entity("API.Entities.AppUserBookmark", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AppUserId") + .HasColumnType("INTEGER"); + + b.Property("ChapterId") + .HasColumnType("INTEGER"); + + b.Property("Created") + .HasColumnType("TEXT"); + + b.Property("FileName") + .HasColumnType("TEXT"); + + b.Property("LastModified") + .HasColumnType("TEXT"); + + b.Property("Page") + .HasColumnType("INTEGER"); + + b.Property("SeriesId") + .HasColumnType("INTEGER"); + + b.Property("VolumeId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AppUserId"); + + b.ToTable("AppUserBookmark"); + }); + + modelBuilder.Entity("API.Entities.AppUserPreferences", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AppUserId") + .HasColumnType("INTEGER"); + + b.Property("AutoCloseMenu") + .HasColumnType("INTEGER"); + + b.Property("BackgroundColor") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasDefaultValue("#000000"); + + b.Property("BlurUnreadSummaries") + .HasColumnType("INTEGER"); + + b.Property("BookReaderFontFamily") + .HasColumnType("TEXT"); + + b.Property("BookReaderFontSize") + .HasColumnType("INTEGER"); + + b.Property("BookReaderImmersiveMode") + .HasColumnType("INTEGER"); + + b.Property("BookReaderLayoutMode") + .HasColumnType("INTEGER"); + + b.Property("BookReaderLineSpacing") + .HasColumnType("INTEGER"); + + b.Property("BookReaderMargin") + .HasColumnType("INTEGER"); + + b.Property("BookReaderReadingDirection") + .HasColumnType("INTEGER"); + + b.Property("BookReaderTapToPaginate") + .HasColumnType("INTEGER"); + + b.Property("BookThemeName") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasDefaultValue("Dark"); + + b.Property("GlobalPageLayoutMode") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasDefaultValue(0); + + b.Property("LayoutMode") + .HasColumnType("INTEGER"); + + b.Property("PageSplitOption") + .HasColumnType("INTEGER"); + + b.Property("PromptForDownloadSize") + .HasColumnType("INTEGER"); + + b.Property("ReaderMode") + .HasColumnType("INTEGER"); + + b.Property("ReadingDirection") + .HasColumnType("INTEGER"); + + b.Property("ScalingOption") + .HasColumnType("INTEGER"); + + b.Property("ShowScreenHints") + .HasColumnType("INTEGER"); + + b.Property("ThemeId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AppUserId") + .IsUnique(); + + b.HasIndex("ThemeId"); + + b.ToTable("AppUserPreferences"); + }); + + modelBuilder.Entity("API.Entities.AppUserProgress", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AppUserId") + .HasColumnType("INTEGER"); + + b.Property("BookScrollId") + .HasColumnType("TEXT"); + + b.Property("ChapterId") + .HasColumnType("INTEGER"); + + b.Property("Created") + .HasColumnType("TEXT"); + + b.Property("LastModified") + .HasColumnType("TEXT"); + + b.Property("PagesRead") + .HasColumnType("INTEGER"); + + b.Property("SeriesId") + .HasColumnType("INTEGER"); + + b.Property("VolumeId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AppUserId"); + + b.HasIndex("SeriesId"); + + b.ToTable("AppUserProgresses"); + }); + + modelBuilder.Entity("API.Entities.AppUserRating", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AppUserId") + .HasColumnType("INTEGER"); + + b.Property("Rating") + .HasColumnType("INTEGER"); + + b.Property("Review") + .HasColumnType("TEXT"); + + b.Property("SeriesId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AppUserId"); + + b.HasIndex("SeriesId"); + + b.ToTable("AppUserRating"); + }); + + modelBuilder.Entity("API.Entities.AppUserRole", b => + { + b.Property("UserId") + .HasColumnType("INTEGER"); + + b.Property("RoleId") + .HasColumnType("INTEGER"); + + b.HasKey("UserId", "RoleId"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetUserRoles", (string)null); + }); + + modelBuilder.Entity("API.Entities.Chapter", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AgeRating") + .HasColumnType("INTEGER"); + + b.Property("AvgHoursToRead") + .HasColumnType("INTEGER"); + + b.Property("Count") + .HasColumnType("INTEGER"); + + b.Property("CoverImage") + .HasColumnType("TEXT"); + + b.Property("CoverImageLocked") + .HasColumnType("INTEGER"); + + b.Property("Created") + .HasColumnType("TEXT"); + + b.Property("IsSpecial") + .HasColumnType("INTEGER"); + + b.Property("Language") + .HasColumnType("TEXT"); + + b.Property("LastModified") + .HasColumnType("TEXT"); + + b.Property("MaxHoursToRead") + .HasColumnType("INTEGER"); + + b.Property("MinHoursToRead") + .HasColumnType("INTEGER"); + + b.Property("Number") + .HasColumnType("TEXT"); + + b.Property("Pages") + .HasColumnType("INTEGER"); + + b.Property("Range") + .HasColumnType("TEXT"); + + b.Property("ReleaseDate") + .HasColumnType("TEXT"); + + b.Property("Summary") + .HasColumnType("TEXT"); + + b.Property("Title") + .HasColumnType("TEXT"); + + b.Property("TitleName") + .HasColumnType("TEXT"); + + b.Property("TotalCount") + .HasColumnType("INTEGER"); + + b.Property("VolumeId") + .HasColumnType("INTEGER"); + + b.Property("WordCount") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("VolumeId"); + + b.ToTable("Chapter"); + }); + + modelBuilder.Entity("API.Entities.CollectionTag", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CoverImage") + .HasColumnType("TEXT"); + + b.Property("CoverImageLocked") + .HasColumnType("INTEGER"); + + b.Property("NormalizedTitle") + .HasColumnType("TEXT"); + + b.Property("Promoted") + .HasColumnType("INTEGER"); + + b.Property("RowVersion") + .HasColumnType("INTEGER"); + + b.Property("Summary") + .HasColumnType("TEXT"); + + b.Property("Title") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("Id", "Promoted") + .IsUnique(); + + b.ToTable("CollectionTag"); + }); + + modelBuilder.Entity("API.Entities.FolderPath", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("LastScanned") + .HasColumnType("TEXT"); + + b.Property("LibraryId") + .HasColumnType("INTEGER"); + + b.Property("Path") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("LibraryId"); + + b.ToTable("FolderPath"); + }); + + modelBuilder.Entity("API.Entities.Genre", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ExternalTag") + .HasColumnType("INTEGER"); + + b.Property("NormalizedTitle") + .HasColumnType("TEXT"); + + b.Property("Title") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedTitle", "ExternalTag") + .IsUnique(); + + b.ToTable("Genre"); + }); + + modelBuilder.Entity("API.Entities.Library", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CoverImage") + .HasColumnType("TEXT"); + + b.Property("Created") + .HasColumnType("TEXT"); + + b.Property("LastModified") + .HasColumnType("TEXT"); + + b.Property("LastScanned") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("Library"); + }); + + modelBuilder.Entity("API.Entities.MangaFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ChapterId") + .HasColumnType("INTEGER"); + + b.Property("Created") + .HasColumnType("TEXT"); + + b.Property("FilePath") + .HasColumnType("TEXT"); + + b.Property("Format") + .HasColumnType("INTEGER"); + + b.Property("LastFileAnalysis") + .HasColumnType("TEXT"); + + b.Property("LastModified") + .HasColumnType("TEXT"); + + b.Property("Pages") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("ChapterId"); + + b.ToTable("MangaFile"); + }); + + modelBuilder.Entity("API.Entities.Metadata.SeriesMetadata", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AgeRating") + .HasColumnType("INTEGER"); + + b.Property("AgeRatingLocked") + .HasColumnType("INTEGER"); + + b.Property("CharacterLocked") + .HasColumnType("INTEGER"); + + b.Property("ColoristLocked") + .HasColumnType("INTEGER"); + + b.Property("CoverArtistLocked") + .HasColumnType("INTEGER"); + + b.Property("EditorLocked") + .HasColumnType("INTEGER"); + + b.Property("GenresLocked") + .HasColumnType("INTEGER"); + + b.Property("InkerLocked") + .HasColumnType("INTEGER"); + + b.Property("Language") + .HasColumnType("TEXT"); + + b.Property("LanguageLocked") + .HasColumnType("INTEGER"); + + b.Property("LettererLocked") + .HasColumnType("INTEGER"); + + b.Property("MaxCount") + .HasColumnType("INTEGER"); + + b.Property("PencillerLocked") + .HasColumnType("INTEGER"); + + b.Property("PublicationStatus") + .HasColumnType("INTEGER"); + + b.Property("PublicationStatusLocked") + .HasColumnType("INTEGER"); + + b.Property("PublisherLocked") + .HasColumnType("INTEGER"); + + b.Property("ReleaseYear") + .HasColumnType("INTEGER"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .HasColumnType("INTEGER"); + + b.Property("SeriesId") + .HasColumnType("INTEGER"); + + b.Property("Summary") + .HasColumnType("TEXT"); + + b.Property("SummaryLocked") + .HasColumnType("INTEGER"); + + b.Property("TagsLocked") + .HasColumnType("INTEGER"); + + b.Property("TotalCount") + .HasColumnType("INTEGER"); + + b.Property("TranslatorLocked") + .HasColumnType("INTEGER"); + + b.Property("WriterLocked") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("SeriesId") + .IsUnique(); + + b.HasIndex("Id", "SeriesId") + .IsUnique(); + + b.ToTable("SeriesMetadata"); + }); + + modelBuilder.Entity("API.Entities.Metadata.SeriesRelation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("RelationKind") + .HasColumnType("INTEGER"); + + b.Property("SeriesId") + .HasColumnType("INTEGER"); + + b.Property("TargetSeriesId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("SeriesId"); + + b.HasIndex("TargetSeriesId"); + + b.ToTable("SeriesRelation"); + }); + + modelBuilder.Entity("API.Entities.Person", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("NormalizedName") + .HasColumnType("TEXT"); + + b.Property("Role") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("Person"); + }); + + modelBuilder.Entity("API.Entities.ReadingList", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AppUserId") + .HasColumnType("INTEGER"); + + b.Property("CoverImage") + .HasColumnType("TEXT"); + + b.Property("CoverImageLocked") + .HasColumnType("INTEGER"); + + b.Property("Created") + .HasColumnType("TEXT"); + + b.Property("LastModified") + .HasColumnType("TEXT"); + + b.Property("NormalizedTitle") + .HasColumnType("TEXT"); + + b.Property("Promoted") + .HasColumnType("INTEGER"); + + b.Property("Summary") + .HasColumnType("TEXT"); + + b.Property("Title") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("AppUserId"); + + b.ToTable("ReadingList"); + }); + + modelBuilder.Entity("API.Entities.ReadingListItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ChapterId") + .HasColumnType("INTEGER"); + + b.Property("Order") + .HasColumnType("INTEGER"); + + b.Property("ReadingListId") + .HasColumnType("INTEGER"); + + b.Property("SeriesId") + .HasColumnType("INTEGER"); + + b.Property("VolumeId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("ChapterId"); + + b.HasIndex("ReadingListId"); + + b.HasIndex("SeriesId"); + + b.HasIndex("VolumeId"); + + b.ToTable("ReadingListItem"); + }); + + modelBuilder.Entity("API.Entities.Series", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AppUserId") + .HasColumnType("INTEGER"); + + b.Property("AvgHoursToRead") + .HasColumnType("INTEGER"); + + b.Property("CoverImage") + .HasColumnType("TEXT"); + + b.Property("CoverImageLocked") + .HasColumnType("INTEGER"); + + b.Property("Created") + .HasColumnType("TEXT"); + + b.Property("FolderPath") + .HasColumnType("TEXT"); + + b.Property("Format") + .HasColumnType("INTEGER"); + + b.Property("LastChapterAdded") + .HasColumnType("TEXT"); + + b.Property("LastFolderScanned") + .HasColumnType("TEXT"); + + b.Property("LastModified") + .HasColumnType("TEXT"); + + b.Property("LibraryId") + .HasColumnType("INTEGER"); + + b.Property("LocalizedName") + .HasColumnType("TEXT"); + + b.Property("LocalizedNameLocked") + .HasColumnType("INTEGER"); + + b.Property("MaxHoursToRead") + .HasColumnType("INTEGER"); + + b.Property("MinHoursToRead") + .HasColumnType("INTEGER"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("NameLocked") + .HasColumnType("INTEGER"); + + b.Property("NormalizedLocalizedName") + .HasColumnType("TEXT"); + + b.Property("NormalizedName") + .HasColumnType("TEXT"); + + b.Property("OriginalName") + .HasColumnType("TEXT"); + + b.Property("Pages") + .HasColumnType("INTEGER"); + + b.Property("SortName") + .HasColumnType("TEXT"); + + b.Property("SortNameLocked") + .HasColumnType("INTEGER"); + + b.Property("WordCount") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AppUserId"); + + b.HasIndex("LibraryId"); + + b.ToTable("Series"); + }); + + modelBuilder.Entity("API.Entities.ServerSetting", b => + { + b.Property("Key") + .HasColumnType("INTEGER"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .HasColumnType("INTEGER"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("Key"); + + b.ToTable("ServerSetting"); + }); + + modelBuilder.Entity("API.Entities.SiteTheme", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Created") + .HasColumnType("TEXT"); + + b.Property("FileName") + .HasColumnType("TEXT"); + + b.Property("IsDefault") + .HasColumnType("INTEGER"); + + b.Property("LastModified") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("NormalizedName") + .HasColumnType("TEXT"); + + b.Property("Provider") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("SiteTheme"); + }); + + modelBuilder.Entity("API.Entities.Tag", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ExternalTag") + .HasColumnType("INTEGER"); + + b.Property("NormalizedTitle") + .HasColumnType("TEXT"); + + b.Property("Title") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedTitle", "ExternalTag") + .IsUnique(); + + b.ToTable("Tag"); + }); + + modelBuilder.Entity("API.Entities.Volume", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AvgHoursToRead") + .HasColumnType("INTEGER"); + + b.Property("CoverImage") + .HasColumnType("TEXT"); + + b.Property("Created") + .HasColumnType("TEXT"); + + b.Property("LastModified") + .HasColumnType("TEXT"); + + b.Property("MaxHoursToRead") + .HasColumnType("INTEGER"); + + b.Property("MinHoursToRead") + .HasColumnType("INTEGER"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("Number") + .HasColumnType("INTEGER"); + + b.Property("Pages") + .HasColumnType("INTEGER"); + + b.Property("SeriesId") + .HasColumnType("INTEGER"); + + b.Property("WordCount") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("SeriesId"); + + b.ToTable("Volume"); + }); + + modelBuilder.Entity("AppUserLibrary", b => + { + b.Property("AppUsersId") + .HasColumnType("INTEGER"); + + b.Property("LibrariesId") + .HasColumnType("INTEGER"); + + b.HasKey("AppUsersId", "LibrariesId"); + + b.HasIndex("LibrariesId"); + + b.ToTable("AppUserLibrary"); + }); + + modelBuilder.Entity("ChapterGenre", b => + { + b.Property("ChaptersId") + .HasColumnType("INTEGER"); + + b.Property("GenresId") + .HasColumnType("INTEGER"); + + b.HasKey("ChaptersId", "GenresId"); + + b.HasIndex("GenresId"); + + b.ToTable("ChapterGenre"); + }); + + modelBuilder.Entity("ChapterPerson", b => + { + b.Property("ChapterMetadatasId") + .HasColumnType("INTEGER"); + + b.Property("PeopleId") + .HasColumnType("INTEGER"); + + b.HasKey("ChapterMetadatasId", "PeopleId"); + + b.HasIndex("PeopleId"); + + b.ToTable("ChapterPerson"); + }); + + modelBuilder.Entity("ChapterTag", b => + { + b.Property("ChaptersId") + .HasColumnType("INTEGER"); + + b.Property("TagsId") + .HasColumnType("INTEGER"); + + b.HasKey("ChaptersId", "TagsId"); + + b.HasIndex("TagsId"); + + b.ToTable("ChapterTag"); + }); + + modelBuilder.Entity("CollectionTagSeriesMetadata", b => + { + b.Property("CollectionTagsId") + .HasColumnType("INTEGER"); + + b.Property("SeriesMetadatasId") + .HasColumnType("INTEGER"); + + b.HasKey("CollectionTagsId", "SeriesMetadatasId"); + + b.HasIndex("SeriesMetadatasId"); + + b.ToTable("CollectionTagSeriesMetadata"); + }); + + modelBuilder.Entity("GenreSeriesMetadata", b => + { + b.Property("GenresId") + .HasColumnType("INTEGER"); + + b.Property("SeriesMetadatasId") + .HasColumnType("INTEGER"); + + b.HasKey("GenresId", "SeriesMetadatasId"); + + b.HasIndex("SeriesMetadatasId"); + + b.ToTable("GenreSeriesMetadata"); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ClaimType") + .HasColumnType("TEXT"); + + b.Property("ClaimValue") + .HasColumnType("TEXT"); + + b.Property("RoleId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("RoleId"); + + b.ToTable("AspNetRoleClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ClaimType") + .HasColumnType("TEXT"); + + b.Property("ClaimValue") + .HasColumnType("TEXT"); + + b.Property("UserId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserClaims", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.Property("LoginProvider") + .HasColumnType("TEXT"); + + b.Property("ProviderKey") + .HasColumnType("TEXT"); + + b.Property("ProviderDisplayName") + .HasColumnType("TEXT"); + + b.Property("UserId") + .HasColumnType("INTEGER"); + + b.HasKey("LoginProvider", "ProviderKey"); + + b.HasIndex("UserId"); + + b.ToTable("AspNetUserLogins", (string)null); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.Property("UserId") + .HasColumnType("INTEGER"); + + b.Property("LoginProvider") + .HasColumnType("TEXT"); + + b.Property("Name") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("UserId", "LoginProvider", "Name"); + + b.ToTable("AspNetUserTokens", (string)null); + }); + + modelBuilder.Entity("PersonSeriesMetadata", b => + { + b.Property("PeopleId") + .HasColumnType("INTEGER"); + + b.Property("SeriesMetadatasId") + .HasColumnType("INTEGER"); + + b.HasKey("PeopleId", "SeriesMetadatasId"); + + b.HasIndex("SeriesMetadatasId"); + + b.ToTable("PersonSeriesMetadata"); + }); + + modelBuilder.Entity("SeriesMetadataTag", b => + { + b.Property("SeriesMetadatasId") + .HasColumnType("INTEGER"); + + b.Property("TagsId") + .HasColumnType("INTEGER"); + + b.HasKey("SeriesMetadatasId", "TagsId"); + + b.HasIndex("TagsId"); + + b.ToTable("SeriesMetadataTag"); + }); + + modelBuilder.Entity("API.Entities.AppUserBookmark", b => + { + b.HasOne("API.Entities.AppUser", "AppUser") + .WithMany("Bookmarks") + .HasForeignKey("AppUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AppUser"); + }); + + modelBuilder.Entity("API.Entities.AppUserPreferences", b => + { + b.HasOne("API.Entities.AppUser", "AppUser") + .WithOne("UserPreferences") + .HasForeignKey("API.Entities.AppUserPreferences", "AppUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.SiteTheme", "Theme") + .WithMany() + .HasForeignKey("ThemeId"); + + b.Navigation("AppUser"); + + b.Navigation("Theme"); + }); + + modelBuilder.Entity("API.Entities.AppUserProgress", b => + { + b.HasOne("API.Entities.AppUser", "AppUser") + .WithMany("Progresses") + .HasForeignKey("AppUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.Series", null) + .WithMany("Progress") + .HasForeignKey("SeriesId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AppUser"); + }); + + modelBuilder.Entity("API.Entities.AppUserRating", b => + { + b.HasOne("API.Entities.AppUser", "AppUser") + .WithMany("Ratings") + .HasForeignKey("AppUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.Series", null) + .WithMany("Ratings") + .HasForeignKey("SeriesId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AppUser"); + }); + + modelBuilder.Entity("API.Entities.AppUserRole", b => + { + b.HasOne("API.Entities.AppRole", "Role") + .WithMany("UserRoles") + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.AppUser", "User") + .WithMany("UserRoles") + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Role"); + + b.Navigation("User"); + }); + + modelBuilder.Entity("API.Entities.Chapter", b => + { + b.HasOne("API.Entities.Volume", "Volume") + .WithMany("Chapters") + .HasForeignKey("VolumeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Volume"); + }); + + modelBuilder.Entity("API.Entities.FolderPath", b => + { + b.HasOne("API.Entities.Library", "Library") + .WithMany("Folders") + .HasForeignKey("LibraryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Library"); + }); + + modelBuilder.Entity("API.Entities.MangaFile", b => + { + b.HasOne("API.Entities.Chapter", "Chapter") + .WithMany("Files") + .HasForeignKey("ChapterId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Chapter"); + }); + + modelBuilder.Entity("API.Entities.Metadata.SeriesMetadata", b => + { + b.HasOne("API.Entities.Series", "Series") + .WithOne("Metadata") + .HasForeignKey("API.Entities.Metadata.SeriesMetadata", "SeriesId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Series"); + }); + + modelBuilder.Entity("API.Entities.Metadata.SeriesRelation", b => + { + b.HasOne("API.Entities.Series", "Series") + .WithMany("Relations") + .HasForeignKey("SeriesId") + .OnDelete(DeleteBehavior.ClientCascade) + .IsRequired(); + + b.HasOne("API.Entities.Series", "TargetSeries") + .WithMany("RelationOf") + .HasForeignKey("TargetSeriesId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Series"); + + b.Navigation("TargetSeries"); + }); + + modelBuilder.Entity("API.Entities.ReadingList", b => + { + b.HasOne("API.Entities.AppUser", "AppUser") + .WithMany("ReadingLists") + .HasForeignKey("AppUserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AppUser"); + }); + + modelBuilder.Entity("API.Entities.ReadingListItem", b => + { + b.HasOne("API.Entities.Chapter", "Chapter") + .WithMany() + .HasForeignKey("ChapterId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.ReadingList", "ReadingList") + .WithMany("Items") + .HasForeignKey("ReadingListId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.Series", "Series") + .WithMany() + .HasForeignKey("SeriesId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.Volume", "Volume") + .WithMany() + .HasForeignKey("VolumeId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Chapter"); + + b.Navigation("ReadingList"); + + b.Navigation("Series"); + + b.Navigation("Volume"); + }); + + modelBuilder.Entity("API.Entities.Series", b => + { + b.HasOne("API.Entities.AppUser", null) + .WithMany("WantToRead") + .HasForeignKey("AppUserId"); + + b.HasOne("API.Entities.Library", "Library") + .WithMany("Series") + .HasForeignKey("LibraryId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Library"); + }); + + modelBuilder.Entity("API.Entities.Volume", b => + { + b.HasOne("API.Entities.Series", "Series") + .WithMany("Volumes") + .HasForeignKey("SeriesId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Series"); + }); + + modelBuilder.Entity("AppUserLibrary", b => + { + b.HasOne("API.Entities.AppUser", null) + .WithMany() + .HasForeignKey("AppUsersId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.Library", null) + .WithMany() + .HasForeignKey("LibrariesId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ChapterGenre", b => + { + b.HasOne("API.Entities.Chapter", null) + .WithMany() + .HasForeignKey("ChaptersId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.Genre", null) + .WithMany() + .HasForeignKey("GenresId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ChapterPerson", b => + { + b.HasOne("API.Entities.Chapter", null) + .WithMany() + .HasForeignKey("ChapterMetadatasId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.Person", null) + .WithMany() + .HasForeignKey("PeopleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ChapterTag", b => + { + b.HasOne("API.Entities.Chapter", null) + .WithMany() + .HasForeignKey("ChaptersId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.Tag", null) + .WithMany() + .HasForeignKey("TagsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("CollectionTagSeriesMetadata", b => + { + b.HasOne("API.Entities.CollectionTag", null) + .WithMany() + .HasForeignKey("CollectionTagsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.Metadata.SeriesMetadata", null) + .WithMany() + .HasForeignKey("SeriesMetadatasId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("GenreSeriesMetadata", b => + { + b.HasOne("API.Entities.Genre", null) + .WithMany() + .HasForeignKey("GenresId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.Metadata.SeriesMetadata", null) + .WithMany() + .HasForeignKey("SeriesMetadatasId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim", b => + { + b.HasOne("API.Entities.AppRole", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserClaim", b => + { + b.HasOne("API.Entities.AppUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserLogin", b => + { + b.HasOne("API.Entities.AppUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityUserToken", b => + { + b.HasOne("API.Entities.AppUser", null) + .WithMany() + .HasForeignKey("UserId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("PersonSeriesMetadata", b => + { + b.HasOne("API.Entities.Person", null) + .WithMany() + .HasForeignKey("PeopleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.Metadata.SeriesMetadata", null) + .WithMany() + .HasForeignKey("SeriesMetadatasId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("SeriesMetadataTag", b => + { + b.HasOne("API.Entities.Metadata.SeriesMetadata", null) + .WithMany() + .HasForeignKey("SeriesMetadatasId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("API.Entities.Tag", null) + .WithMany() + .HasForeignKey("TagsId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("API.Entities.AppRole", b => + { + b.Navigation("UserRoles"); + }); + + modelBuilder.Entity("API.Entities.AppUser", b => + { + b.Navigation("Bookmarks"); + + b.Navigation("Progresses"); + + b.Navigation("Ratings"); + + b.Navigation("ReadingLists"); + + b.Navigation("UserPreferences"); + + b.Navigation("UserRoles"); + + b.Navigation("WantToRead"); + }); + + modelBuilder.Entity("API.Entities.Chapter", b => + { + b.Navigation("Files"); + }); + + modelBuilder.Entity("API.Entities.Library", b => + { + b.Navigation("Folders"); + + b.Navigation("Series"); + }); + + modelBuilder.Entity("API.Entities.ReadingList", b => + { + b.Navigation("Items"); + }); + + modelBuilder.Entity("API.Entities.Series", b => + { + b.Navigation("Metadata"); + + b.Navigation("Progress"); + + b.Navigation("Ratings"); + + b.Navigation("RelationOf"); + + b.Navigation("Relations"); + + b.Navigation("Volumes"); + }); + + modelBuilder.Entity("API.Entities.Volume", b => + { + b.Navigation("Chapters"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/API/Data/Migrations/20220819223212_NormalizedLocalizedName.cs b/API/Data/Migrations/20220819223212_NormalizedLocalizedName.cs new file mode 100644 index 0000000000..600a3a6b22 --- /dev/null +++ b/API/Data/Migrations/20220819223212_NormalizedLocalizedName.cs @@ -0,0 +1,25 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace API.Data.Migrations +{ + public partial class NormalizedLocalizedName : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "NormalizedLocalizedName", + table: "Series", + type: "TEXT", + nullable: true); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "NormalizedLocalizedName", + table: "Series"); + } + } +} diff --git a/API/Data/Migrations/DataContextModelSnapshot.cs b/API/Data/Migrations/DataContextModelSnapshot.cs index a3cdf7f05c..d65cc4adbc 100644 --- a/API/Data/Migrations/DataContextModelSnapshot.cs +++ b/API/Data/Migrations/DataContextModelSnapshot.cs @@ -818,6 +818,9 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("NameLocked") .HasColumnType("INTEGER"); + b.Property("NormalizedLocalizedName") + .HasColumnType("TEXT"); + b.Property("NormalizedName") .HasColumnType("TEXT"); diff --git a/API/Data/Repositories/SeriesRepository.cs b/API/Data/Repositories/SeriesRepository.cs index 3d0d4a5de5..528d46902a 100644 --- a/API/Data/Repositories/SeriesRepository.cs +++ b/API/Data/Repositories/SeriesRepository.cs @@ -121,6 +121,7 @@ public interface ISeriesRepository Task GetSeriesIdByFolder(string folder); Task GetSeriesByFolderPath(string folder); Task GetFullSeriesByName(string series, int libraryId); + Task GetFullSeriesByAnyName(string seriesName, string localizedName, int libraryId); Task RemoveSeriesNotInList(IList seenSeries, int libraryId); Task>> GetFolderPathMap(int libraryId); } @@ -1168,10 +1169,65 @@ public async Task GetSeriesByFolderPath(string folder) return await _context.Series.SingleOrDefaultAsync(s => s.FolderPath.Equals(normalized)); } + /// + /// Finds a series by series name for a given library. + /// + /// This pulls everything with the Series, so should be used only when needing tracking on all related tables + /// + /// + /// public Task GetFullSeriesByName(string series, int libraryId) { + var localizedSeries = Parser.Parser.Normalize(series); + return _context.Series + .Where(s => (s.NormalizedName.Equals(localizedSeries) + || s.LocalizedName.Equals(series)) && s.LibraryId == libraryId) + .Include(s => s.Metadata) + .ThenInclude(m => m.People) + .Include(s => s.Metadata) + .ThenInclude(m => m.Genres) + .Include(s => s.Library) + .Include(s => s.Volumes) + .ThenInclude(v => v.Chapters) + .ThenInclude(cm => cm.People) + + .Include(s => s.Volumes) + .ThenInclude(v => v.Chapters) + .ThenInclude(c => c.Tags) + + .Include(s => s.Volumes) + .ThenInclude(v => v.Chapters) + .ThenInclude(c => c.Genres) + + + .Include(s => s.Metadata) + .ThenInclude(m => m.Tags) + + .Include(s => s.Volumes) + .ThenInclude(v => v.Chapters) + .ThenInclude(c => c.Files) + .AsSplitQuery() + .SingleOrDefaultAsync(); + } + + /// + /// Finds a series by series name or localized name for a given library. + /// + /// This pulls everything with the Series, so should be used only when needing tracking on all related tables + /// + /// + /// + /// + public Task GetFullSeriesByAnyName(string seriesName, string localizedName, int libraryId) + { + var localizedSeries = Parser.Parser.Normalize(seriesName); + var normalizedLocalized = Parser.Parser.Normalize(localizedName); return _context.Series - .Where(s => s.NormalizedName.Equals(Parser.Parser.Normalize(series)) && s.LibraryId == libraryId) + .Where(s => s.NormalizedName.Equals(localizedSeries) + || s.NormalizedName.Equals(normalizedLocalized) + || s.NormalizedLocalizedName.Equals(localizedSeries) + || s.NormalizedLocalizedName.Equals(normalizedLocalized)) + .Where(s => s.LibraryId == libraryId) .Include(s => s.Metadata) .ThenInclude(m => m.People) .Include(s => s.Metadata) diff --git a/API/Entities/Series.cs b/API/Entities/Series.cs index 00e7dd33df..b343240c02 100644 --- a/API/Entities/Series.cs +++ b/API/Entities/Series.cs @@ -18,6 +18,10 @@ public class Series : IEntityDate, IHasReadTimeEstimate /// public string NormalizedName { get; set; } /// + /// Used internally for localized name matching. + /// + public string NormalizedLocalizedName { get; set; } + /// /// The name used to sort the Series. By default, will be the same as Name. /// public string SortName { get; set; } diff --git a/API/Parser/Parser.cs b/API/Parser/Parser.cs index 8c0cfec51a..35594caa38 100644 --- a/API/Parser/Parser.cs +++ b/API/Parser/Parser.cs @@ -1033,9 +1033,15 @@ public static bool IsCoverImage(string filename) return IsImage(filename) && CoverImageRegex.IsMatch(filename); } + /// + /// Validates that a Path doesn't start with certain blacklisted folders, like __MACOSX, @Recently-Snapshot, etc and that if a full path, the filename + /// doesn't start with ._, which is a metadata file on MACOSX. + /// + /// + /// public static bool HasBlacklistedFolderInPath(string path) { - return path.Contains("__MACOSX") || path.StartsWith("@Recently-Snapshot") || path.StartsWith("@recycle") || path.StartsWith("._") || path.Contains(".qpkg"); + return path.Contains("__MACOSX") || path.StartsWith("@Recently-Snapshot") || path.StartsWith("@recycle") || path.StartsWith("._") || Path.GetFileName(path).StartsWith("._") || path.Contains(".qpkg"); } diff --git a/API/Services/DirectoryService.cs b/API/Services/DirectoryService.cs index 5dde7f750a..8c7d29603f 100644 --- a/API/Services/DirectoryService.cs +++ b/API/Services/DirectoryService.cs @@ -641,17 +641,11 @@ public IList ScanFiles(string folderPath, GlobMatcher? matcher = null) /// /// Recursively scans a folder and returns the max last write time on any folders /// - /// This is required vs just an attribute check as NTFS does not bubble up certain events from nested folders. - /// This will also ignore recursive nature if the device is not NTFS /// /// Max Last Write Time public DateTime GetLastWriteTime(string folderPath) { if (!FileSystem.Directory.Exists(folderPath)) throw new IOException($"{folderPath} does not exist"); - if (new DriveInfo(FileSystem.Path.GetPathRoot(folderPath)).DriveFormat != "NTFS") - { - return FileSystem.Directory.GetLastWriteTime(folderPath); - } var directories = GetAllDirectories(folderPath).ToList(); if (directories.Count == 0) return FileSystem.Directory.GetLastWriteTime(folderPath); diff --git a/API/Services/SeriesService.cs b/API/Services/SeriesService.cs index c51aa887d7..db3b1d1d64 100644 --- a/API/Services/SeriesService.cs +++ b/API/Services/SeriesService.cs @@ -8,7 +8,6 @@ using API.DTOs; using API.DTOs.CollectionTags; using API.DTOs.Metadata; -using API.DTOs.Reader; using API.DTOs.SeriesDetail; using API.Entities; using API.Entities.Enums; diff --git a/API/Services/Tasks/Scanner/ProcessSeries.cs b/API/Services/Tasks/Scanner/ProcessSeries.cs index 5cb5e357fa..29b9cab1d2 100644 --- a/API/Services/Tasks/Scanner/ProcessSeries.cs +++ b/API/Services/Tasks/Scanner/ProcessSeries.cs @@ -79,29 +79,47 @@ public async Task ProcessSeriesAsync(IList parsedInfos, Library libr { if (!parsedInfos.Any()) return; + var seriesAdded = false; var scanWatch = Stopwatch.StartNew(); var seriesName = parsedInfos.First().Series; - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Updated, seriesName)); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, + MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Updated, seriesName)); _logger.LogInformation("[ScannerService] Beginning series update on {SeriesName}", seriesName); // Check if there is a Series - var seriesAdded = false; - var series = await _unitOfWork.SeriesRepository.GetFullSeriesByName(parsedInfos.First().Series, library.Id); + var firstInfo = parsedInfos.First(); + Series series = null; + try + { + series = + await _unitOfWork.SeriesRepository.GetFullSeriesByAnyName(firstInfo.Series, firstInfo.LocalizedSeries, + library.Id); + } + catch (Exception ex) + { + _logger.LogError(ex, "There was an exception finding existing series for {SeriesName} with Localized name of {LocalizedName}. This indicates you have duplicate series with same name or localized name in the library. Correct this and rescan", firstInfo.Series, firstInfo.LocalizedSeries); + return; + } + if (series == null) { seriesAdded = true; - series = DbFactory.Series(parsedInfos.First().Series); + series = DbFactory.Series(firstInfo.Series); + series.LocalizedName = firstInfo.LocalizedSeries; } + if (series.LibraryId == 0) series.LibraryId = library.Id; try { + _logger.LogInformation("[ScannerService] Processing series {SeriesName}", series.OriginalName); UpdateVolumes(series, parsedInfos); series.Pages = series.Volumes.Sum(v => v.Pages); series.NormalizedName = Parser.Parser.Normalize(series.Name); + series.NormalizedLocalizedName = Parser.Parser.Normalize(series.LocalizedName); series.OriginalName ??= parsedInfos[0].Series; if (series.Format == MangaFormat.Unknown) { diff --git a/API/Services/Tasks/ScannerService.cs b/API/Services/Tasks/ScannerService.cs index e0d250fa46..332270fef2 100644 --- a/API/Services/Tasks/ScannerService.cs +++ b/API/Services/Tasks/ScannerService.cs @@ -198,6 +198,8 @@ void TrackFiles(Tuple> parsedInfo) await Task.WhenAll(processTasks); + // At this point, we've already inserted the series into the DB OR we haven't and seenSeries has our series + // We now need to do any leftover work, like removing // We need to handle if parsedSeries is empty but seenSeries has our series if (seenSeries.Any(s => s.NormalizedName.Equals(series.NormalizedName)) && parsedSeries.Keys.Count == 0) { @@ -208,65 +210,56 @@ await _eventHub.SendMessageAsync(MessageFactory.Info, "All folders have not been changed since last scan. Scan will be aborted.")); _processSeries.EnqueuePostSeriesProcessTasks(series.LibraryId, seriesId, false); + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name)); return; } + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name)); + // Remove any parsedSeries keys that don't belong to our series. This can occur when users store 2 series in the same folder RemoveParsedInfosNotForSeries(parsedSeries, series); - // If nothing was found, first validate any of the files still exist. If they don't then we have a deletion and can skip the rest of the logic flow - if (parsedSeries.Count == 0) - { - var anyFilesExist = - (await _unitOfWork.SeriesRepository.GetFilesForSeries(series.Id)).Any(m => File.Exists(m.FilePath)); - - if (!anyFilesExist) - { - try - { - _unitOfWork.SeriesRepository.Remove(series); - await CommitAndSend(1, sw, scanElapsedTime, series); - } - catch (Exception ex) - { - _logger.LogCritical(ex, "There was an error during ScanSeries to delete the series as no files could be found. Aborting scan"); - await _unitOfWork.RollbackAsync(); - return; - } - } - else - { - // I think we should just fail and tell user to fix their setup. This is extremely expensive for an edge case - _logger.LogCritical("We weren't able to find any files in the series scan, but there should be. Please correct your naming convention or put Series in a dedicated folder. Aborting scan"); - await _eventHub.SendMessageAsync(MessageFactory.Error, - MessageFactory.ErrorEvent($"Error scanning {series.Name}", "We weren't able to find any files in the series scan, but there should be. Please correct your naming convention or put Series in a dedicated folder. Aborting scan")); - await _unitOfWork.RollbackAsync(); - return; - } - // At this point, parsedSeries will have at least one key and we can perform the update. If it still doesn't, just return and don't do anything - if (parsedSeries.Count == 0) return; - } - - - try - { - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name)); - var parsedInfos = ParseScannedFiles.GetInfosByName(parsedSeries, series); - await _processSeries.ProcessSeriesAsync(parsedInfos, library); - await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name)); - - await CommitAndSend(1, sw, scanElapsedTime, series); - } - catch (Exception ex) - { - _logger.LogCritical(ex, "There was an error during ScanSeries to update the series"); - await _unitOfWork.RollbackAsync(); - } + // If nothing was found, first validate any of the files still exist. If they don't then we have a deletion and can skip the rest of the logic flow + if (parsedSeries.Count == 0) + { + var anyFilesExist = + (await _unitOfWork.SeriesRepository.GetFilesForSeries(series.Id)).Any(m => File.Exists(m.FilePath)); + + if (!anyFilesExist) + { + try + { + _unitOfWork.SeriesRepository.Remove(series); + await CommitAndSend(1, sw, scanElapsedTime, series); + await _eventHub.SendMessageAsync(MessageFactory.SeriesRemoved, + MessageFactory.SeriesRemovedEvent(seriesId, string.Empty, series.LibraryId), false); + } + catch (Exception ex) + { + _logger.LogCritical(ex, "There was an error during ScanSeries to delete the series as no files could be found. Aborting scan"); + await _unitOfWork.RollbackAsync(); + return; + } + } + else + { + // I think we should just fail and tell user to fix their setup. This is extremely expensive for an edge case + _logger.LogCritical("We weren't able to find any files in the series scan, but there should be. Please correct your naming convention or put Series in a dedicated folder. Aborting scan"); + await _eventHub.SendMessageAsync(MessageFactory.Error, + MessageFactory.ErrorEvent($"Error scanning {series.Name}", "We weren't able to find any files in the series scan, but there should be. Please correct your naming convention or put Series in a dedicated folder. Aborting scan")); + await _unitOfWork.RollbackAsync(); + return; + } + // At this point, parsedSeries will have at least one key and we can perform the update. If it still doesn't, just return and don't do anything + if (parsedSeries.Count == 0) return; + } + + + await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name)); // Tell UI that this series is done await _eventHub.SendMessageAsync(MessageFactory.ScanSeries, MessageFactory.ScanSeriesEvent(library.Id, seriesId, series.Name)); - await _metadataService.RemoveAbandonedMetadataKeys(); BackgroundJob.Enqueue(() => _cacheService.CleanupChapters(chapterIds)); BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory)); diff --git a/API/Startup.cs b/API/Startup.cs index 768e46f8df..cb9e14acaf 100644 --- a/API/Startup.cs +++ b/API/Startup.cs @@ -176,6 +176,7 @@ public void Configure(IApplicationBuilder app, IBackgroundJobClient backgroundJo var logger = serviceProvider.GetRequiredService>(); var userManager = serviceProvider.GetRequiredService>(); var themeService = serviceProvider.GetRequiredService(); + var dataContext = serviceProvider.GetRequiredService(); await MigrateBookmarks.Migrate(directoryService, unitOfWork, logger, cacheService); @@ -185,6 +186,9 @@ await MigrateBookmarks.Migrate(directoryService, unitOfWork, await MigrateRemoveExtraThemes.Migrate(unitOfWork, themeService); + // Only needed for v0.5.5.x and v0.5.6 + await MigrateNormalizedLocalizedName.Migrate(unitOfWork, dataContext, logger); + // Update the version in the DB after all migrations are run var installVersion = await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.InstallVersion); installVersion.Value = BuildInfo.Version.ToString(); diff --git a/UI/Web/src/app/_services/account.service.ts b/UI/Web/src/app/_services/account.service.ts index f59583a525..04809a4dc7 100644 --- a/UI/Web/src/app/_services/account.service.ts +++ b/UI/Web/src/app/_services/account.service.ts @@ -228,8 +228,7 @@ export class AccountService implements OnDestroy { private refreshToken() { if (this.currentUser === null || this.currentUser === undefined) return of(); - //console.log('refreshing token and updating user account'); - + return this.httpClient.post<{token: string, refreshToken: string}>(this.baseUrl + 'account/refresh-token', {token: this.currentUser.token, refreshToken: this.currentUser.refreshToken}).pipe(map(user => { if (this.currentUser) { this.currentUser.token = user.token; diff --git a/UI/Web/src/app/nav/events-widget/events-widget.component.html b/UI/Web/src/app/nav/events-widget/events-widget.component.html index 051b55d786..6b2f999849 100644 --- a/UI/Web/src/app/nav/events-widget/events-widget.component.html +++ b/UI/Web/src/app/nav/events-widget/events-widget.component.html @@ -165,7 +165,7 @@
  • {{onlineUsers.length}} Users online
    • -
  • Not much going on here
    • +
  • Not much going on here
  • Active Events: {{activeEvents}}