Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parser optimization part1 #1531

Merged
merged 8 commits into from
Sep 18, 2022
Merged

Parser optimization part1 #1531

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2de9cbd
Optimize CleanTitle
tjarls Sep 1, 2022
814a2a6
Optimize MangaEditionRegex
tjarls Sep 1, 2022
c27ed49
Optimize special regexes
tjarls Sep 1, 2022
f05ff69
Refactor manga|comic special parsing into simple tests
tjarls Sep 2, 2022
4d3110f
Word bind the special regexps. Support additional "special" use cases.
tjarls Sep 14, 2022
064963b
Updates to address PR comments
tjarls Sep 15, 2022
f209e7e
CleanTitle benchmarking
tjarls Sep 16, 2022
ee9a1c2
Use a smaller Comics Data set for benchmarking
tjarls Sep 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions API.Tests/Parser/ComicParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,8 +197,12 @@ public void ParseComicChapterTest(string filename, string expected)
[InlineData("Adventure Time 2013 Annual #001 (2013)", true)]
[InlineData("Adventure Time 2013_Annual_#001 (2013)", true)]
[InlineData("Adventure Time 2013_-_Annual #001 (2013)", true)]
public void ParseComicSpecialTest(string input, bool expected)
[InlineData("G.I. Joe - A Real American Hero Yearbook 004 Reprint (2021)", false)]
[InlineData("Mazebook 001", false)]
[InlineData("X-23 One Shot (2010)", true)]
[InlineData("Casus Belli v1 Hors-Série 21 - Mousquetaires et Sorcellerie", true)]
public void IsComicSpecialTest(string input, bool expected)
{
Assert.Equal(expected, !string.IsNullOrEmpty(API.Services.Tasks.Scanner.Parser.Parser.ParseComicSpecial(input)));
Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.IsComicSpecial(input));
}
}
15 changes: 5 additions & 10 deletions API.Tests/Parser/MangaParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,7 @@ public void ParseChaptersTest(string filename, string expected)
[InlineData("Wotakoi - Love is Hard for Otaku Omnibus v01 (2018) (Digital) (danke-Empire)", "Omnibus")]
[InlineData("To Love Ru v01 Uncensored (Ch.001-007)", "Uncensored")]
[InlineData("Chobits Omnibus Edition v01 [Dark Horse]", "Omnibus Edition")]
[InlineData("Chobits_Omnibus_Edition_v01_[Dark_Horse]", "Omnibus Edition")]
[InlineData("[dmntsf.net] One Piece - Digital Colored Comics Vol. 20 Ch. 177 - 30 Million vs 81 Million.cbz", "")]
[InlineData("AKIRA - c003 (v01) [Full Color] [Darkhorse].cbz", "")]
[InlineData("Love Hina Omnibus v05 (2015) (Digital-HD) (Asgard-Empire).cbz", "Omnibus")]
Expand All @@ -306,9 +307,11 @@ public void ParseEditionTest(string input, string expected)
[InlineData("Beastars SP01", false)]
[InlineData("The League of Extraordinary Gentlemen", false)]
[InlineData("The League of Extra-ordinary Gentlemen", false)]
public void ParseMangaSpecialTest(string input, bool expected)
[InlineData("Gifting The Wonderful World With Blessings! - 3 Side Stories [yuNS][Unknown].epub", true)]
[InlineData("Dr. Ramune - Mysterious Disease Specialist v01 (2020) (Digital) (danke-Empire).cbz", false)]
public void IsMangaSpecialTest(string input, bool expected)
{
Assert.Equal(expected, !string.IsNullOrEmpty(API.Services.Tasks.Scanner.Parser.Parser.ParseMangaSpecial(input)));
Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.IsMangaSpecial(input));
}

[Theory]
Expand All @@ -320,13 +323,5 @@ public void ParseFormatTest(string inputFile, MangaFormat expected)
Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.ParseFormat(inputFile));
}

[Theory]
[InlineData("Gifting The Wonderful World With Blessings! - 3 Side Stories [yuNS][Unknown].epub", "Side Stories")]
public void ParseSpecialTest(string inputFile, string expected)
{
Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.ParseMangaSpecial(inputFile));
}



}
3 changes: 3 additions & 0 deletions API.Tests/Parser/ParserTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ public void PadZerosTest(string input, string expected)
[InlineData("[Suihei Kiki]_Kasumi_Otoko_no_Ko_[Taruby]_v1.1", false, "Kasumi Otoko no Ko v1.1")]
[InlineData("Batman - Detective Comics - Rebirth Deluxe Edition Book 04 (2019) (digital) (Son of Ultron-Empire)", true, "Batman - Detective Comics - Rebirth Deluxe Edition")]
[InlineData("Something - Full Color Edition", false, "Something - Full Color Edition")]
[InlineData("Witchblade 089 (2005) (Bittertek-DCP) (Top Cow (Image Comics))", true, "Witchblade 089")]
[InlineData("(C99) Kami-sama Hiroimashita. (SSSS.GRIDMAN)", false, "Kami-sama Hiroimashita.")]
[InlineData("Dr. Ramune - Mysterious Disease Specialist v01 (2020) (Digital) (danke-Empire)", false, "Dr. Ramune - Mysterious Disease Specialist v01")]
public void CleanTitleTest(string input, bool isComic, string expected)
{
Assert.Equal(expected, CleanTitle(input, isComic));
Expand Down
6 changes: 3 additions & 3 deletions API/Services/Tasks/Scanner/Parser/DefaultParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,10 @@ public ParserInfo Parse(string filePath, string rootPath, LibraryType type = Lib
ret.Edition = edition;
}

var isSpecial = type == LibraryType.Comic ? Services.Tasks.Scanner.Parser.Parser.ParseComicSpecial(fileName) : Services.Tasks.Scanner.Parser.Parser.ParseMangaSpecial(fileName);
bool isSpecial = type == LibraryType.Comic ? Services.Tasks.Scanner.Parser.Parser.IsComicSpecial(fileName) : Services.Tasks.Scanner.Parser.Parser.IsMangaSpecial(fileName);
// We must ensure that we can only parse a special out. As some files will have v20 c171-180+Omake and that
// could cause a problem as Omake is a special term, but there is valid volume/chapter information.
if (ret.Chapters == Services.Tasks.Scanner.Parser.Parser.DefaultChapter && ret.Volumes == Services.Tasks.Scanner.Parser.Parser.DefaultVolume && !string.IsNullOrEmpty(isSpecial))
if (ret.Chapters == Services.Tasks.Scanner.Parser.Parser.DefaultChapter && ret.Volumes == Services.Tasks.Scanner.Parser.Parser.DefaultVolume && isSpecial)
{
ret.IsSpecial = true;
ParseFromFallbackFolders(filePath, rootPath, type, ref ret); // NOTE: This can cause some complications, we should try to be a bit less aggressive to fallback to folder
Expand Down Expand Up @@ -131,7 +131,7 @@ public void ParseFromFallbackFolders(string filePath, string rootPath, LibraryTy
for (var i = 0; i < fallbackFolders.Count; i++)
{
var folder = fallbackFolders[i];
if (!string.IsNullOrEmpty(Services.Tasks.Scanner.Parser.Parser.ParseMangaSpecial(folder))) continue;
if (Services.Tasks.Scanner.Parser.Parser.IsMangaSpecial(folder)) continue;

var parsedVolume = type is LibraryType.Manga ? Services.Tasks.Scanner.Parser.Parser.ParseVolume(folder) : Services.Tasks.Scanner.Parser.Parser.ParseComicVolume(folder);
var parsedChapter = type is LibraryType.Manga ? Services.Tasks.Scanner.Parser.Parser.ParseChapter(folder) : Services.Tasks.Scanner.Parser.Parser.ParseComicChapter(folder);
Expand Down
Loading