Optimizing Sitemap Creation with Batched Content Items (#16636)

---------

Co-authored-by: Sébastien Ros <sebastienros@gmail.com>

src/OrchardCore.Modules/OrchardCore.ContentLocalization/Sitemaps/LocalizedContentItemsQueryProvider.cs

@@ -1,5 +1,3 @@
-using System.Linq;
-using System.Threading.Tasks;
 using OrchardCore.ContentLocalization.Models;
 using OrchardCore.ContentLocalization.Records;
 using OrchardCore.ContentManagement;
@@ -11,114 +9,111 @@
 using YesSql;
 using YesSql.Services;
 
-namespace OrchardCore.ContentLocalization.Sitemaps
+namespace OrchardCore.ContentLocalization.Sitemaps;
+
+public class LocalizedContentItemsQueryProvider : IContentItemsQueryProvider
 {
-    public class LocalizedContentItemsQueryProvider : IContentItemsQueryProvider
+    private readonly IStore _store;
+    private readonly IRouteableContentTypeCoordinator _routeableContentTypeCoordinator;
+    private readonly ILocalizationService _localizationService;
+
+    public LocalizedContentItemsQueryProvider(
+        IStore store,
+        IRouteableContentTypeCoordinator routeableContentTypeCoordinator,
+        ILocalizationService localizationService
+        )
     {
-        private readonly ISession _session;
-        private readonly IRouteableContentTypeCoordinator _routeableContentTypeCoordinator;
-        private readonly ILocalizationService _localizationService;
-
-        public LocalizedContentItemsQueryProvider(
-            ISession session,
-            IRouteableContentTypeCoordinator routeableContentTypeCoordinator,
-            ILocalizationService localizationService
-            )
-        {
-            _session = session;
-            _routeableContentTypeCoordinator = routeableContentTypeCoordinator;
-            _localizationService = localizationService;
-        }
+        _store = store;
+        _routeableContentTypeCoordinator = routeableContentTypeCoordinator;
+        _localizationService = localizationService;
+    }
 
-        public async Task GetContentItemsAsync(ContentTypesSitemapSource source, ContentItemsQueryContext queryContext)
-        {
-            var routeableContentTypeDefinitions = await _routeableContentTypeCoordinator.ListRoutableTypeDefinitionsAsync();
+    public async Task GetContentItemsAsync(ContentTypesSitemapSource source, ContentItemsQueryContext context, int? skip = null, int? take = null)
+    {
+        var routeableContentTypeDefinitions = await _routeableContentTypeCoordinator.ListRoutableTypeDefinitionsAsync();
+        using var session = _store.CreateSession(withTracking: false);
 
-            if (source.IndexAll)
-            {
-                // Assumption here is that at least one content type will be localized.
-                var ctdNames = routeableContentTypeDefinitions.Select(ctd => ctd.Name);
+        IEnumerable<ContentItem> contentItems = null;
 
-                var queryResults = await _session.Query<ContentItem>()
-                    .With<ContentItemIndex>(x => x.Published && x.ContentType.IsIn(ctdNames))
-                    .OrderBy(x => x.CreatedUtc)
-                    .ListAsync();
+        if (source.IndexAll)
+        {
+            // Assumption here is that at least one content type will be localized.
+            var ctdNames = routeableContentTypeDefinitions.Select(ctd => ctd.Name);
 
-                queryContext.ContentItems = queryResults;
+            contentItems = await session.Query<ContentItem>()
+                .With<ContentItemIndex>(x => x.Published && x.ContentType.IsIn(ctdNames))
+                .OrderBy(x => x.CreatedUtc)
+                .ThenBy(x => x.Id)
+                .Skip(skip ?? 0)
+                .Take(take ?? 0)
+                .ListAsync();
+
+        }
+        else if (source.LimitItems)
+        {
+            // Test that content type is still valid to include in sitemap.
+            var contentType = routeableContentTypeDefinitions
+                .FirstOrDefault(ctd => string.Equals(source.LimitedContentType.ContentTypeName, ctd.Name, StringComparison.Ordinal));
 
-                // Provide all content items with localization as reference content items.
-                queryContext.ReferenceContentItems = queryResults
-                    .Where(ci => ci.Has<LocalizationPart>());
+            if (contentType == null)
+            {
+                return;
             }
-            else if (source.LimitItems)
+
+            if (contentType.Parts.Any(ctd => string.Equals(ctd.Name, nameof(LocalizationPart), StringComparison.Ordinal)))
             {
-                // Test that content type is still valid to include in sitemap.
-                var contentType = routeableContentTypeDefinitions
-                    .FirstOrDefault(ctd => string.Equals(source.LimitedContentType.ContentTypeName, ctd.Name));
-
-                if (contentType == null)
-                {
-                    return;
-                }
-
-                if (contentType.Parts.Any(ctd => string.Equals(ctd.Name, nameof(LocalizationPart))))
-                {
-                    // Get all content items here for reference. Then reduce by default culture.
-                    // We know that the content item should be localized.
-                    // If it doesn't have a localization part, the content item should have been saved.
-                    var queryResults = await _session.Query<ContentItem>()
-                         .With<ContentItemIndex>(ci => ci.ContentType == source.LimitedContentType.ContentTypeName && ci.Published)
-                         .OrderBy(ci => ci.CreatedUtc)
-                         .With<LocalizedContentItemIndex>()
-                         .ListAsync();
-
-                    // When limiting items Content item is valid if it is for the default culture.
-                    var defaultCulture = await _localizationService.GetDefaultCultureAsync();
-
-                    // Reduce by default culture.
-                    var items = queryResults
-                        .Where(ci => string.Equals(ci.As<LocalizationPart>().Culture, defaultCulture))
-                        .Skip(source.LimitedContentType.Skip)
-                        .Take(source.LimitedContentType.Take);
-
-                    queryContext.ContentItems = items;
-
-                    // Provide all content items with localization as reference content items.
-                    queryContext.ReferenceContentItems = queryResults
-                        .Where(ci => ci.Has<LocalizationPart>());
-                }
-                else
-                {
-                    // Content type is not localized. Produce standard results.
-                    var queryResults = await _session.Query<ContentItem>()
-                        .With<ContentItemIndex>(x => x.ContentType == source.LimitedContentType.ContentTypeName && x.Published)
-                        .OrderBy(x => x.CreatedUtc)
-                        .Skip(source.LimitedContentType.Skip)
-                        .Take(source.LimitedContentType.Take)
-                        .ListAsync();
-
-                    queryContext.ContentItems = queryResults;
-                }
+                // When limiting items Content item is valid if it is for the default culture.
+                var defaultCulture = await _localizationService.GetDefaultCultureAsync();
+
+                // Get all content items here for reference. Then reduce by default culture.
+                // We know that the content item should be localized.
+                // If it doesn't have a localization part, the content item should have been saved.
+                contentItems = await session.Query<ContentItem>()
+                    .With<ContentItemIndex>(ci => ci.ContentType == source.LimitedContentType.ContentTypeName && ci.Published)
+                    .OrderBy(ci => ci.CreatedUtc)
+                    .ThenBy(ci => ci.Id)
+                    .With<LocalizedContentItemIndex>(x => x.Culture == defaultCulture)
+                    .Take(take ?? 0)
+                    .Skip(skip ?? 0)
+                    .ListAsync();
             }
             else
             {
-                // Test that content types are still valid to include in sitemap.
-                var typesToIndex = routeableContentTypeDefinitions
-                    .Where(ctd => source.ContentTypes.Any(s => string.Equals(ctd.Name, s.ContentTypeName)))
-                    .Select(x => x.Name);
-
-                // No advantage here in reducing with localized index.
-                var queryResults = await _session.Query<ContentItem>()
-                    .With<ContentItemIndex>(x => x.ContentType.IsIn(typesToIndex) && x.Published)
+                // Content type is not localized. Produce standard results.
+                contentItems = await session.Query<ContentItem>()
+                    .With<ContentItemIndex>(x => x.ContentType == source.LimitedContentType.ContentTypeName && x.Published)
                     .OrderBy(x => x.CreatedUtc)
+                    .Skip(skip ?? 0)
+                    .Take(take ?? 0)
                     .ListAsync();
 
-                queryContext.ContentItems = queryResults;
-
-                // Provide all content items with localization as reference content items.
-                queryContext.ReferenceContentItems = queryResults
-                    .Where(ci => ci.Has<LocalizationPart>());
             }
         }
+        else
+        {
+            // Test that content types are still valid to include in sitemap.
+            var typesToIndex = routeableContentTypeDefinitions
+                .Where(ctd => source.ContentTypes.Any(s => string.Equals(ctd.Name, s.ContentTypeName, StringComparison.Ordinal)))
+                .Select(x => x.Name);
+
+            // No advantage here in reducing with localized index.
+
+            contentItems = await session.Query<ContentItem>()
+                .With<ContentItemIndex>(x => x.ContentType.IsIn(typesToIndex) && x.Published)
+                .OrderBy(x => x.CreatedUtc)
+                .ThenBy(x => x.Id)
+                .Skip(skip ?? 0)
+                .Take(take ?? 0)
+                .ListAsync();
+
+        }
+
+        if (contentItems != null)
+        {
+            context.ContentItems = contentItems;
+
+            // Provide all content items with localization as reference content items.
+            context.ReferenceContentItems = contentItems.Where(ci => ci.Has<LocalizationPart>());
+        }
     }
 }

src/OrchardCore.Modules/OrchardCore.ContentLocalization/Sitemaps/SitemapUrlHrefLangExtendedMetadataProvider.cs

@@ -1,68 +1,67 @@
-using System.Linq;
-using System.Threading.Tasks;
 using System.Xml.Linq;
 using OrchardCore.ContentLocalization.Models;
 using OrchardCore.ContentManagement;
 using OrchardCore.Sitemaps.Aspects;
 using OrchardCore.Sitemaps.Builders;
 using OrchardCore.Sitemaps.Services;
 
-namespace OrchardCore.ContentLocalization.Sitemaps
+namespace OrchardCore.ContentLocalization.Sitemaps;
+
+public class SitemapUrlHrefLangExtendedMetadataProvider : ISitemapContentItemExtendedMetadataProvider
 {
-    public class SitemapUrlHrefLangExtendedMetadataProvider : ISitemapContentItemExtendedMetadataProvider
+    private static readonly XNamespace _extendedNamespace = "http://www.w3.org/1999/xhtml";
+    private static readonly XAttribute _extendedAttribute = new(XNamespace.Xmlns + "xhtml", _extendedNamespace);
+
+    private readonly IContentManager _contentManager;
+    private readonly IRouteableContentTypeCoordinator _routeableContentTypeCoordinator;
+
+    public SitemapUrlHrefLangExtendedMetadataProvider(
+        IContentManager contentManager,
+        IRouteableContentTypeCoordinator routeableContentTypeCoordinator
+        )
     {
-        private static readonly XNamespace _extendedNamespace = "http://www.w3.org/1999/xhtml";
-        private static readonly XAttribute _extendedAttribute = new(XNamespace.Xmlns + "xhtml", _extendedNamespace);
+        _contentManager = contentManager;
+        _routeableContentTypeCoordinator = routeableContentTypeCoordinator;
+    }
 
-        private readonly IContentManager _contentManager;
-        private readonly IRouteableContentTypeCoordinator _routeableContentTypeCoordinator;
+    public XAttribute GetExtendedAttribute => _extendedAttribute;
 
-        public SitemapUrlHrefLangExtendedMetadataProvider(
-            IContentManager contentManager,
-            IRouteableContentTypeCoordinator routeableContentTypeCoordinator
-            )
+    public async Task<bool> ApplyExtendedMetadataAsync(
+        SitemapBuilderContext context,
+        ContentItemsQueryContext queryContext,
+        ContentItem contentItem,
+        XElement url)
+    {
+        var part = contentItem.As<LocalizationPart>();
+        if (part == null ||
+            queryContext.ReferenceContentItems == null ||
+            !queryContext.ReferenceContentItems.Any())
         {
-            _contentManager = contentManager;
-            _routeableContentTypeCoordinator = routeableContentTypeCoordinator;
+            return true;
         }
 
-        public XAttribute GetExtendedAttribute => _extendedAttribute;
+        var localizedContentParts = queryContext.ReferenceContentItems
+            .Select(ci => ci.As<LocalizationPart>())
+            .Where(cp => cp.LocalizationSet == part.LocalizationSet);
 
-        public async Task<bool> ApplyExtendedMetadataAsync(
-            SitemapBuilderContext context,
-            ContentItemsQueryContext queryContext,
-            ContentItem contentItem,
-            XElement url)
+        foreach (var localizedPart in localizedContentParts)
         {
-            var part = contentItem.As<LocalizationPart>();
-            if (part == null)
+            var sitemapMetadataAspect = await _contentManager.PopulateAspectAsync<SitemapMetadataAspect>(localizedPart.ContentItem);
+            if (sitemapMetadataAspect.Exclude)
             {
-                return true;
+                continue;
             }
 
-            var localizedContentParts = queryContext.ReferenceContentItems
-                .Select(ci => ci.As<LocalizationPart>())
-                .Where(cp => cp.LocalizationSet == part.LocalizationSet);
+            var hrefValue = await _routeableContentTypeCoordinator.GetRouteAsync(context, localizedPart.ContentItem);
 
-            foreach (var localizedPart in localizedContentParts)
-            {
-                var sitemapMetadataAspect = await _contentManager.PopulateAspectAsync<SitemapMetadataAspect>(localizedPart.ContentItem);
-                if (sitemapMetadataAspect.Exclude)
-                {
-                    continue;
-                }
+            var linkNode = new XElement(_extendedNamespace + "link",
+                new XAttribute("rel", "alternate"),
+                new XAttribute("hreflang", localizedPart.Culture),
+                new XAttribute("href", hrefValue));
 
-                var hrefValue = await _routeableContentTypeCoordinator.GetRouteAsync(context, localizedPart.ContentItem);
-
-                var linkNode = new XElement(_extendedNamespace + "link",
-                    new XAttribute("rel", "alternate"),
-                    new XAttribute("hreflang", localizedPart.Culture),
-                    new XAttribute("href", hrefValue));
-
-                url.Add(linkNode);
-            }
-
-            return true;
+            url.Add(linkNode);
         }
+
+        return true;
     }
 }