handle entry sorting in mongo (#772)

sort entries in mongo instead of server side to reduce memory usage
sillsdev · May 6, 2024 · 3e12b8a · 3e12b8a
1 parent b0d7857
commit 3e12b8a
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 16 deletions.
diff --git a/backend/LfClassicData/LfClassicLexboxApi.cs b/backend/LfClassicData/LfClassicLexboxApi.cs
@@ -30,8 +30,13 @@ public async Task<WritingSystems> GetWritingSystems()
  Abbreviation = inputSystem.Abbreviation
  };
  //ws type might not be stored, we will add it anyway, otherwise nothing works
- if (inputSystem.AnalysisWS is not false) analysis.Add(writingSystem);
- if (inputSystem.VernacularWS is not false) vernacular.Add(writingSystem);
+ if (inputSystem is { AnalysisWS: null, VernacularWS: null })
+ {
+ analysis.Add(writingSystem);
+ vernacular.Add(writingSystem);
+ }
+ if (inputSystem.AnalysisWS is true) analysis.Add(writingSystem);
+ if (inputSystem.VernacularWS is true) vernacular.Add(writingSystem);
  }
  return new WritingSystems
  {
@@ -59,29 +64,28 @@ public IAsyncEnumerable<Entry> SearchEntries(string query, QueryOptions? options
  private async IAsyncEnumerable<Entry> Query(QueryOptions? options = null)
  {
  options ??= QueryOptions.Default;
- var ws = await GetWritingSystems();
- if (ws is { Vernacular: [], Analysis: [] })
- {
- yield break;
- }
+
  var sortWs = options.Order.WritingSystem;
  if (sortWs == "default")
  {
+ var ws = await GetWritingSystems();
+ if (ws is { Vernacular: [], Analysis: [] })
+ {
+ yield break;
+ }
  sortWs = ws.Vernacular[0].Id;
  }
 
- await foreach (var entry in Entries.ToAsyncEnumerable()
+ await foreach (var entry in Entries.AsQueryable()
  //todo, you can only sort by headword for now
- .OrderBy(e => e.CitationForm?.TryGetValue(sortWs, out var val) == true
- ? val.Value
- : e.Lexeme?.TryGetValue(sortWs, out val) == true
- ? val.Value
- : string.Empty)
- .ThenBy(e => e.MorphologyType)
- .ThenBy(e => e.Guid)//todo should sort by homograph number
+ .Select(entry => new {entry, headword = entry.CitationForm![sortWs].Value ?? entry.Lexeme![sortWs].Value ?? string.Empty})
+ .OrderBy(e => e.headword)
+ .ThenBy(e => e.entry.MorphologyType)
+ .ThenBy(e => e.entry.Guid) //todo should sort by homograph number
  .Skip(options.Offset)
  .Take(options.Count)
- .Select(ToEntry))
+ .ToAsyncEnumerable()
+ .Select(e => ToEntry(e.entry)))
  {
  yield return entry;
  }

diff --git a/backend/LfClassicData/MongoExtensions.cs b/backend/LfClassicData/MongoExtensions.cs
@@ -15,4 +15,15 @@ public static async IAsyncEnumerable<T> ToAsyncEnumerable<T>(this IMongoCollecti
  }
  }
  }
+ public static async IAsyncEnumerable<T> ToAsyncEnumerable<T>(this IAsyncCursorSource<T> cursorSource)
+ {
+ using var entriesCursor = await cursorSource.ToCursorAsync();
+ while (await entriesCursor.MoveNextAsync())
+ {
+ foreach (var entry in entriesCursor.Current)
+ {
+ yield return entry;
+ }
+ }
+ }
 }