How to add support for literate languages?

[MangelMaxime]

Hello,

I am looking into eleventy for my blog and documentation project. And I am missing a way to support F# literate file.

F# literate file, is a way to structure a file with documentation in mind. This is really useful as you can write your documentation page using a real F# file meaning that the compiler is checking that your code is valid etc.

Note: Here I am using F# as an example, but others language also support it

It can looks like that:

(***
title: This is post written using F# literate
layout: layouts/base.njk
***)

// This is a code block
let value = "Hello World"

(***
# This a documentation comment

Lorem ipsum dolor sit amet, consectetur adipiscing elit.
Donec euismod, nisl eget consectetur consectetur, nisl nunc egestas nisi, eu consectetur nunc nisi euismod nunc.

***)

// More code block below

let add (a : int) (b : int) = a + b

once converted into Markdown it looks like that:

---
title: This is post written using F# literate
layout: layouts/base.njk
---

```fs
// This is a code block
let value = "Hello World"
```

# This a documentation comment

Lorem ipsum dolor sit amet, consectetur adipiscing elit.
Donec euismod, nisl eget consectetur consectetur, nisl nunc egestas nisi, eu consectetur nunc nisi euismod nunc.

```fs
// More code block below

let add (a : int) (b : int) = a + b
```

I didn't find a way to pre-process the content of a file so I tried to write my own plugin for eleventy but ended up with 2 problems.

Problem 1: Front matter delimiters

In order, to write valid F# file I need to write the front matter using customs delimiters otherwise the compiler will give a syntax error.

(***
title: This is post written using F# literate
layout: layouts/base.njk
***)

I did that using

// Custom front matter delimiters to use valid F# code
eleventyConfig.setFrontMatterParsingOptions({
    delimiters: ["(***", "***)"],
});

But this means that for all the files delimiters are now (*** and ***) instead of ---. This gives a degraded user experience because syntax highlighting or extension doesn't work correctly because they don't detect the new front matter syntax.

Plus, if I am using other literate languages I will probably need another type of limiter to be supported.

Is it possible to support multiple style of front matter delimiters?

Problem 2: Nunjucks instruction are not processed correctly

If I add lines like that in the .fsx file

{% set lastModifiedDate = page.inputPath | lastModifiedDate | formatDate %}

<i>Last updated on <strong>{{ lastModifiedDate }}</strong></i>

then they are treated as raw content and not process as nunjucks instructions.

Is there a way to have nunjucks instructions to be process like done for the markdown files?

---

Here are the configuration I am using:

.eleventy.js

const remarkVscode = require("gatsby-remark-vscode");
const rehypeRaw = require("rehype-raw");
const rehypeStringify = require("rehype-stringify");
const remarkToRehype = require("remark-rehype");
const eleventyFsharpLiterate = require("./_11ty/plugins/fsharp-literate");

const remarkOptions = {
    enableRehype: false,
    plugins: [
        {
            plugin: remarkVscode.remarkPlugin,
            options: {
                inlineCode: {
                    marker: "•",
                },
            },
        },
        {
            plugin: remarkToRehype,
            options: {
                allowDangerousHtml: true,
            },
        },
        rehypeRaw,
        {
            plugin: rehypeStringify,
            options: {
                allowDangerousHtml: true,
                closeSelfClosing: true,
            },
        },
    ],
};

module.exports = function (eleventyConfig) {

    eleventyConfig.addPlugin(eleventyFsharpLiterate, remarkOptions);

    // set input and output folder
    return {
        dir: {
            input: "src",
        },
        dataTemplateEngine: "njk",
        markdownTemplateEngine: "njk",
    };
};

_11ty/plugins/fsharp-literate.js

const eleventyRemarkInternal = require("@fec/eleventy-plugin-remark/src/eleventyRemark");
const collect = require("collect.js");

const defaultEleventyRemarkOptions = {
    plugins: [],
    enableRehype: true,
};

const startMarkdownCommentRegex = /\(\*(\*)+/;
const endMarkdownCommentRegex = /(\*)+\)/;

/**
 * Skip the empty lines at the beginning of the collection
 * it stop at the first non-empty line
 */
collect().macro("skipEmptyLinesAtTheBeginning", function () {
    return this.skipWhile((line) => line === "");
});

/**
 * Skip empty lines at the end of the collection
 */
collect().macro("skipEmptyLinesAtTheEnd", function () {
    return this.reverse().skipEmptyLinesAtTheBeginning().reverse();
});

collect().macro("add", function (item) {
    this.items.unshift(item);

    return this;
});

/**
 *
 * @param {collect.Collection} result - Processed lines
 * @param {collect.Collection} lines - Content to process
 */
function processFile(result, lines) {
    const line = lines.shift();

    if (line === null) {
        return result.join("\n");
    }

    const trimmedLine = line.trim();

    if (trimmedLine === "(*** hide ***)") {
        const rest = lines.skipUntil((line) => line.trim().startsWith("(**"));

        return processFile(result, rest);
    } else if (trimmedLine === "(*** show ***)") {
        // Skip this instruction line
        return processFile(result, lines);
    } else if (trimmedLine.startsWith("(**")) {
        // This is the start of an F# markdown comment
        // Strip the markdown comment start delimiter
        result.push(line.replace(startMarkdownCommentRegex, ""));

        const markdownLines = lines.takeUntil((line) => {
            return line.trimEnd().endsWith("*)");
        });

        const rest = lines.skip(markdownLines.count());

        const newResult = result.concat(markdownLines);

        const endLine = rest.first();

        // Check if the line ending the markdown comment has some content
        // If yes, capture it and add it to the result
        if (endLine) {
            newResult.push(endLine.replace(endMarkdownCommentRegex, ""));
        }

        return processFile(newResult, rest);
    } else {
        const codeLines = lines.takeUntil((line) => {
            return line.trim().startsWith("(**");
        });

        const rest = lines.skip(codeLines.count());

        // Remove non meaningful lines
        const sanetizedCodeLines = codeLines
            .skipEmptyLinesAtTheBeginning()
            .skipEmptyLinesAtTheEnd();

        let actualCode = collect();

        // If there are actual code line add them inside of a code block
        if (sanetizedCodeLines.isNotEmpty()) {
            actualCode = collect(["```fs"])
                .concat(sanetizedCodeLines)
                .concat(["```"]);
        }

        const newResult = result.concat(actualCode);

        return processFile(newResult, rest);
    }
}

function transformFsxToMarkdown(fileContent) {
    const lines = fileContent.replace(/\r\n/g, "\n").split("\n");

    // Remove empty lines at the beginning of the file
    const sanetizedLines = collect(lines).skipEmptyLinesAtTheBeginning();

    return processFile(collect(), sanetizedLines);
}

const fsharpExtension = (pluginOptions) => {
    const options = Object.assign(
        {},
        defaultEleventyRemarkOptions,
        pluginOptions
    );

    return {
        compile: async (inputContent) => {
            return async (data) => {
                const remark = eleventyRemarkInternal(options);
                const markdownContent = transformFsxToMarkdown(inputContent);
                return remark.render(markdownContent, data);
            };
        },
    };
};

module.exports = {
    initArguments: {},
    configFunction: (eleventyConfig, pluginOptions = {}) => {
        // Custom front matter delimiters to use valid F# code
        eleventyConfig.setFrontMatterParsingOptions({
            delimiters: ["(***", "***)"],
        });

        // Add F# file support to eleventy
        eleventyConfig.addTemplateFormats("fsx");
        eleventyConfig.addTemplateFormats("fs");

        // Configure how to handle fsx files
        eleventyConfig.addExtension("fsx", fsharpExtension(pluginOptions));
        eleventyConfig.addExtension("fs", fsharpExtension(pluginOptions));
    },
};

[j-f1]

There are two separate issues here:

1. Writing a custom data extractor. If you define the getData method in your custom template language, you can do whatever you want to extract the data from the file (such as parsing it again and running a YAML parser on the comment’s content, or running it as an F# file and looking for a data export)
2. Passing the Markdown data through a template language. If you’re just looking for basic variable substitution, you could pass the compiled Markdown and the data cascade object into a simple engine like Handlebars (although you’d lose the ability to use shortcodes). I don’t know how to use the built-in template engines directly, though.

[MangelMaxime]

Hello,

thank you for looking into it.

I fail to see what problem getData will solved?

If this is related to the front-matter problem, the problem with front-matter is that we can only define one set of delimiters which are going to be applied to all the files.

There doesn't seem to be a way to defined both --- and the pair (*** ***) as delimiters.

Passing the Markdown data through a template language. If you’re just looking for basic variable substitution, you could pass the compiled Markdown and the data cascade object into a simple engine like Handlebars (although you’d lose the ability to use short-codes). I don’t know how to use the built-in template engines directly, though.

In my case, I want the short-codes to be supported otherwise, the "last modifier short codes" is not available and so the literate pages have the raw short code instead of the actual value.

[j-f1]

getData will allow you to handle parsing the data yourself (e.g. by grabbing the content of the first (*** ***) comment and running a YAML parser on it).

In my case, I want the short-codes to be supported otherwise, the "last modifier short codes" is not available and so the literate pages have the raw short code instead of the actual value.

Try enabling the render plugin, and then use this.config.javascriptFunctions.renderTemplate to render the content?

[MangelMaxime]

Using your inputs and a package which add new template format eleventy-plugin-asciidoc, I was able to solve the front matter issue.

Templating is still not fully working because if I have shortcode or filters in the file content it is not processed.

File content:

(***
title: This is my first post.
description: This is a post on My Blog about agile frameworks.
layout: layouts/fsharp.njk
***)

Shortcode/filter directly from the F# file: {{ pageBodyClass }}

<br/><br/>

{% renderTemplate "md" %}
# I am a title

* I am a list
* I am a list
{% endrenderTemplate %}

Result:

The last problem is that it is not rendered as markdown, but I can always process it from the plugin. But the user will need to provide twice the markdown configuration if he changed it.

---

Using eleventyConfig.javascriptFunctions.renderTemplate didn't help, the nunjucks instruction inside the F# file are not processed.

[MangelMaxime]

New plugin sources:

// Code from this file has been based on
// https://github.com/saneef/eleventy-plugin-asciidoc/blob/da3ba916808fdb87e97f25c4451f3a78f55a5769/lib/eleventy-asciidoc.js

// @ts-check

const debug = require("debug")("eleventy-plugin-fsharp-literate");
const fs = require("fs");
const matter = require("gray-matter");
const nunjucks = require("nunjucks");
var MarkdownIt = require('markdown-it');

/** @typedef {import('gray-matter').GrayMatterFile}  GrayMatterFile */

/**
 * Reads a file synchronously.
 *
 * @param      {string}  inputPath  The input path
 * @return     {GrayMatterFile}  { description_of_the_return_value }
 */
const readFileSync = (inputPath) => {
    return matter(fs.readFileSync(inputPath, "utf8"), {
        delimiters: ["(***", "***)"],
    });
};

/**
 * Gets front-matter data from the file synchronously
 *
 * @param      {string}  inputPath  The input path
 * @return     {{ [key: string]: any }}  The data.
 */
const getData = (inputPath) => {
    const { data } = readFileSync(inputPath);

    return {
        // fsharpAttributes: attributes,
        ...data,
    };
};

function eleventyFsharpLiterate(eleventyConfig) {
    const compile = (str, inputPath) => (data) => {
        // console.log(eleventyConfig);
        if (str) {
            // So if str has a value, it's a permalink (which can be a string or a function)
            debug(`Permalink: ${str}`);
            return typeof str === "function"
                ? str(data)
                : nunjucks.renderString(str, data);
        }

        debug(`Reading ${inputPath}`);
        const { content } = readFileSync(inputPath);

        if (content) {
            debug(`Converting:\n ${content}`);
            // TODO: Move to remark js
            let md = new MarkdownIt();
            return md.render(content);
        }
    };

    return {
        read: false,
        getData,
        compile,
    };
}

/**
 *
 * @param {object} eleventyConfig
 */
function configFunction (eleventyConfig) {
    eleventyConfig.addTemplateFormats("fsx");
    eleventyConfig.addExtension("fsx", eleventyFsharpLiterate(eleventyConfig));
}

module.exports = configFunction;

[MangelMaxime]

I think I found a way to have everything working. But it is clearly really "hacky" ^^

I spawn my own instance of NunjucksEngine and used to compile the result of the markdown transformation.

function eleventyFsharpLiterate(eleventyConfig) {
    const compile = async (str, inputPath) => async (data) => {
        // console.log(eleventyConfig);
        if (str) {
            // So if str has a value, it's a permalink (which can be a string or a function)
            debug(`Permalink: ${str}`);
            return typeof str === "function"
                ? str(data)
                : NunjucksLib.renderString(str, data);
        }

        debug(`Reading ${inputPath}`);
        const { content } = readFileSync(inputPath);

        // HACKS
        // Setup the nunjucks used by eleventy, so we can call it ourself to process the content
        // This is neeed otherwise, the shortcodes are not processed
        // Using the nunjucks engine from eleventy, allows to have it setup correctly "for free"
        let nunjucksEngine = new NunjucksEngine(
            "njk",
            {
                input: eleventyConfig.dir.input,
                includes: eleventyConfig.dir.includes,
            },
            eleventyConfig
        );

        if (content) {
            debug(`Converting:\n ${content}`);
            // TODO: Move to remark js
            let md = new MarkdownIt();
            const markdownText = md.render(content);
            const nunjucksCompileFunc = await nunjucksEngine.compile(markdownText, inputPath);
            return await nunjucksCompileFunc(data);
        }
    };

    return {
        read: false,
        getData,
        compile,
    };
}

With that it means that I can:

1. Get the data using the (*** ... ***) front matter
2. Pre-process the file content to transform the F# literate into markdown code
3. Apply the markdown transformation
4. Call Nunjucks to apply the shortcode/filters
5. Return the result for it to continue the Eleventy pipeline

I think this is a really hacky solution but it seems to do the job.