Merge pull request #11397 from quarto-dev/fix/utf8-categories

listing - Correctly handle non ASCII category and special characters

src/command/render/pandoc.ts

@@ -207,6 +207,7 @@ import {
   BrandFontFile,
   BrandFontGoogle,
 } from "../../resources/types/schema-types.ts";
+import { kFieldCategories } from "../../project/types/website/listing/website-listing-shared.ts";
 
 // in case we are running multiple pandoc processes
 // we need to make sure we capture all of the trace files
@@ -1020,6 +1021,10 @@ export async function runPandoc(
       if (key === kTheme && isRevealjsOutput(options.format.pandoc)) {
         continue;
       }
+      // - categories are handled specifically already for website projects with a metadata override and should not be overridden by user input
+      if (key === kFieldCategories && projectIsWebsite(options.project)) {
+        continue;
+      }
       // perform the override
       pandocMetadata[key] = engineMetadata[key];
     }

src/core/base64.ts

@@ -0,0 +1,7 @@
+export function b64EncodeUnicode(str: string) {
+  return btoa(encodeURIComponent(str));
+}
+
+export function unicodeDecodeB64(str: string) {
+  return decodeURIComponent(atob(str));
+}

src/project/types/website/listing/website-listing-categories.ts

@@ -17,6 +17,7 @@ import {
   ListingDescriptor,
   ListingSharedOptions,
 } from "./website-listing-shared.ts";
+import { b64EncodeUnicode } from "../../../../core/base64.ts";
 
 export function categorySidebar(
   doc: Document,
@@ -117,7 +118,7 @@ function categoryElement(
   categoryEl.classList.add("category");
   categoryEl.setAttribute(
     "data-category",
-    value !== undefined ? btoa(value) : btoa(category),
+    value !== undefined ? b64EncodeUnicode(value) : b64EncodeUnicode(category),
   );
   categoryEl.innerHTML = contents;
   return categoryEl;

src/project/types/website/listing/website-listing-template.ts

@@ -44,6 +44,7 @@ import { formatDate, parsePandocDate } from "../../../../core/date.ts";
 import { truncateText } from "../../../../core/text.ts";
 import { encodeAttributeValue } from "../../../../core/html.ts";
 import { imagePlaceholder, isPlaceHolder } from "./website-listing-read.ts";
+import { b64EncodeUnicode, unicodeDecodeB64 } from "../../../../core/base64.ts";
 
 export const kDateFormat = "date-format";
 
@@ -160,6 +161,12 @@ export function templateMarkdownHandler(
       ejsParams["metadataAttrs"] = reshapedListing.utilities.metadataAttrs;
       ejsParams["templateParams"] = reshapedListing["template-params"];
     }
+    // some custom utils function
+    ejsParams["utils"] = {
+      b64encode: b64EncodeUnicode,
+      b64decode: unicodeDecodeB64,
+    };
+
     return ejsParams;
   };
 
@@ -455,7 +462,7 @@ export function reshapeListing(
     attr["index"] = (index++).toString();
     if (item.categories) {
       const str = (item.categories as string[]).join(",");
-      attr["categories"] = btoa(str);
+      attr["categories"] = b64EncodeUnicode(str);
     }
 
     // Add magic attributes for the sortable values

src/project/types/website/website.ts

@@ -86,6 +86,9 @@ import { formatDate } from "../../../core/date.ts";
 import { projectExtensionPathResolver } from "../../../extension/extension.ts";
 import { websiteDraftPostProcessor } from "./website-draft.ts";
 import { projectDraftMode } from "./website-utils.ts";
+import { kFieldCategories } from "./listing/website-listing-shared.ts";
+import { pandocNativeStr } from "../../../core/pandoc/codegen.ts";
+import { asArray } from "../../../core/array.ts";
 
 export const kSiteTemplateDefault = "default";
 export const kSiteTemplateBlog = "blog";
@@ -157,6 +160,7 @@ export const websiteProjectType: ProjectType = {
       // add some title related variables
       extras.pandoc = extras.pandoc || {};
       extras.metadata = extras.metadata || {};
+      extras.metadataOverride = extras.metadataOverride || {};
 
       // Resolve any giscus information
       resolveFormatForGiscus(project, format);
@@ -196,6 +200,18 @@ export const websiteProjectType: ProjectType = {
         extras.metadata[kPageTitle] = title;
       }
 
+      // categories metadata needs to be escaped from Markdown processing to
+      // avoid +smart applying to it. Categories are expected to be non markdown.
+      // So we provide an override to ensure they are not processed.
+      if (format.metadata[kFieldCategories]) {
+        extras.metadataOverride[kFieldCategories] = asArray(
+          format.metadata[kFieldCategories],
+        ).map(
+          (category) =>
+            pandocNativeStr(category as string).mappedString().value,
+        );
+      }
+
       // html metadata
       extras.html = extras.html || {};
       extras.html[kHtmlPostprocessors] = extras.html[kHtmlPostprocessors] || [];

src/resources/projects/website/listing/item-default.ejs.md

@@ -56,7 +56,7 @@ print(`<div class="metadata-value listing-${field}">${listing.utilities.outputLi
 <% if (fields.includes('categories') && item.categories) { %> 
 <div class="listing-categories">
 <% for (const category of item.categories) { %>
-<div class="listing-category" onclick="window.quartoListingCategory('<%=btoa(category)%>'); return false;"><%= category %></div>
+<div class="listing-category" onclick="window.quartoListingCategory('<%=utils.b64encode(category)%>'); return false;"><%= category %></div>
 <% } %>
 </div>
 <% } %> 

src/resources/projects/website/listing/item-grid.ejs.md

@@ -64,7 +64,7 @@ return !["title", "image", "image-alt", "date", "author", "subtitle", "descripti
 
 <div class="listing-categories">
   <% for (const category of item.categories) { %>
-<div class="listing-category" onclick="window.quartoListingCategory('<%=btoa(category)%>'); return false;"><%= category %></div>
+<div class="listing-category" onclick="window.quartoListingCategory('<%=utils.b64encode(category)%>'); return false;"><%= category %></div>
   <% } %>
 </div>
 

src/resources/projects/website/listing/listing-default.ejs.md

@@ -1,5 +1,5 @@
 :::{.list .quarto-listing-default}
 <% for (const item of items) { %>
-<% partial('item-default.ejs.md', {listing, item }) %>
+<% partial('item-default.ejs.md', {listing, item, utils }) %>
 <% } %>
 :::

src/resources/projects/website/listing/listing-grid.ejs.md

@@ -4,6 +4,6 @@ const cols = listing['grid-columns'];
 
 :::{.list .grid .quarto-listing-cols-<%=cols%>}
 <% for (const item of items) { %>
-<% partial('item-grid.ejs.md', {listing, item }) %>
+<% partial('item-grid.ejs.md', {listing, item, utils }) %>
 <% } %>
 :::

src/resources/projects/website/listing/quarto-listing.js

@@ -16,7 +16,9 @@ window["quarto-listing-loaded"] = () => {
   if (hash) {
     // If there is a category, switch to that
     if (hash.category) {
-      activateCategory(hash.category);
+      // category hash are URI encoded so we need to decode it before processing
+      // so that we can match it with the category element processed in JS
+      activateCategory(decodeURIComponent(hash.category));
     }
     // Paginate a specific listing
     const listingIds = Object.keys(window["quarto-listings"]);
@@ -59,7 +61,10 @@ window.document.addEventListener("DOMContentLoaded", function (_event) {
   );
 
   for (const categoryEl of categoryEls) {
-    const category = atob(categoryEl.getAttribute("data-category"));
+    // category needs to support non ASCII characters
+    const category = decodeURIComponent(
+      atob(categoryEl.getAttribute("data-category"))
+    );
     categoryEl.onclick = () => {
       activateCategory(category);
       setCategoryHash(category);
@@ -209,7 +214,9 @@ function activateCategory(category) {
 
   // Activate this category
   const categoryEl = window.document.querySelector(
-    `.quarto-listing-category .category[data-category='${btoa(category)}']`
+    `.quarto-listing-category .category[data-category='${btoa(
+      encodeURIComponent(category)
+    )}']`
   );
   if (categoryEl) {
     categoryEl.classList.add("active");
@@ -232,7 +239,9 @@ function filterListingCategory(category) {
         list.filter(function (item) {
           const itemValues = item.values();
           if (itemValues.categories !== null) {
-            const categories = atob(itemValues.categories).split(",");
+            const categories = decodeURIComponent(
+              atob(itemValues.categories)
+            ).split(",");
             return categories.includes(category);
           } else {
             return false;

tests/docs/playwright/blog/simple-blog/posts/post-with-code/index.qmd

@@ -2,7 +2,7 @@
 title: "Post With Code"
 author: "Harlow Malloc"
 date: "2024-09-06"
-categories: [news, code, analysis]
+categories: [news, code, analysis, apos'trophe]
 image: "image.jpg"
 ---
 

tests/docs/playwright/blog/simple-blog/posts/welcome/index.qmd

@@ -2,7 +2,7 @@
 title: "Welcome To My Blog"
 author: "Tristan O'Malley"
 date: "2024-09-03"
-categories: [news]
+categories: [news, 'euros (€)', 免疫]
 ---
 
 This is the first post in a Quarto blog. Welcome!

tests/docs/smoke-all/2024/10/23/issue-10829/.gitignore

@@ -0,0 +1,2 @@
+/.quarto/
+_site/

tests/integration/playwright/tests/blog-simple-blog.spec.ts

@@ -1,4 +1,5 @@
 import { expect, test } from "@playwright/test";
+import { getUrl } from "../src/utils";
 
 test('List.js is correctly patch to allow searching with lowercase and uppercase', 
   async ({ page }) => {
@@ -29,4 +30,25 @@ test('Categories link are clickable', async ({ page }) => {
   await page.locator('div').filter({ hasText: /^news$/ }).click();
   await expect(page).toHaveURL(/_site\/index\.html#category=news$/);
   await expect(page.locator(`div.category[data-category="${btoa('news')}"]`)).toHaveClass(/active/);
-});
+});
+
+test('Categories links are clickable', async ({ page }) => {
+  const checkCategoryLink = async (category: string) => {
+    await page.getByRole('link', { name: category }).click();
+    await expect(page).toHaveURL(getUrl(`blog/simple-blog/_site/index.html#category=${encodeURIComponent(category)}`));
+    await expect(page.locator(`div.category[data-category="${btoa(encodeURIComponent(category))}"]`)).toHaveClass(/active/);
+  };
+  // Checking link is working
+  await page.goto('./blog/simple-blog/_site/posts/welcome/');
+  await checkCategoryLink('news');
+  // Including for special characters
+  await page.getByRole('link', { name: 'Welcome To My Blog' }).click();
+  await checkCategoryLink('euros (€)');
+  await page.getByRole('link', { name: 'Welcome To My Blog' }).click();
+  await checkCategoryLink('免疫');
+  await page.goto('./blog/simple-blog/_site/posts/post-with-code/');
+  await checkCategoryLink("apos'trophe");
+  // special check for when a page is not loaded from non root path
+  await page.goto('./blog/simple-blog/_site/posts/welcome/#img-lst');
+  await checkCategoryLink('news');
+});