Add support for pluto (#121)

DemoCard Generation from Pluto Notebooks, no tests and no explicit docs
Co-authored-by: Carlo Lucibello <carlo.lucibello@gmail.com>

.github/workflows/UnitTest.yml

@@ -16,7 +16,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- julia-version: ['1.0', '1', 'nightly']
+ julia-version: ['1.6', '1', 'nightly']
  os: [ubuntu-latest, windows-latest, macOS-latest]
 
  steps:

Project.toml

@@ -12,11 +12,15 @@ ImageCore = "a09fc81d-aa75-5fe9-8630-4744c3626534"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
 Mustache = "ffc61752-8dc7-55ee-8c37-f3e9cdd09e70"
+Pluto = "c3e4b0f8-55cb-11ea-2926-15256bba5781"
+PlutoStaticHTML = "359b1769-a58e-495b-9770-312e911026ad"
 Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
 YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
 
 [compat]
 Documenter = "0.22, 0.23, 0.24, 0.25, 0.26, 0.27"
+Pluto = "^0.19"
+PlutoStaticHTML = "^6"
 FileIO = "1"
 HTTP = "0.6, 0.7, 0.8, 0.9, 1"
 ImageCore = "0.7, 0.8, 0.9"
@@ -25,7 +29,7 @@ Literate = "2"
 Mustache = "0.5, 1.0"
 Suppressor = "0.2"
 YAML = "0.3, 0.4"
-julia = "1"
+julia = "1.6"
 
 [extras]
 ImageMagick = "6218d12a-5da1-5696-b52f-db25d2ecc6d1"

src/DemoCards.jl

@@ -6,6 +6,8 @@ using Mustache
 using Literate
 using ImageCore
 using FileIO, JSON, YAML
+using Pluto
+using PlutoStaticHTML
 using Suppressor # suppress log generated by 3rd party tools, e.g., Literate
 import HTTP
 using Documenter

src/types/card.jl

@@ -23,7 +23,11 @@ function democard(path::String)::AbstractDemoCard
  if ext in markdown_exts
  return MarkdownDemoCard(path)
  elseif ext in julia_exts
- return JuliaDemoCard(path)
+ if is_pluto_notebook(path)
+ return PlutoDemoCard(path)
+ else
+ return JuliaDemoCard(path)
+ end
  else
  return UnmatchedCard(path)
  end
@@ -122,3 +126,4 @@ end
 
 include("markdown.jl")
 include("julia.jl")
+include("pluto.jl")

src/types/pluto.jl

@@ -0,0 +1,178 @@
+const pluto_footer = raw"""
+
+---
+
+*This page was generated using [DemoCards.jl](https://github.com/johnnychen94/DemoCards.jl). and [PlutoStaticHTML.jl](https://github.com/rikhuijzer/PlutoStaticHTML.jl)*
+
+
+"""
+
+"""
+ struct PlutoDemoCard <: AbstractDemoCard
+ PlutoDemoCard(path::AbstractString)
+
+Constructs a markdown-format demo card from a pluto notebook.
+
+
+# Fields
+
+Besides `path`, this struct has some other fields:
+
+* `path`: path to the source markdown file
+* `cover`: path to the cover image
+* `id`: cross-reference id
+* `title`: one-line description of the demo card
+* `author`: author(s) of this demo.
+* `date`: the update date of this demo.
+* `julia`: Julia version compatibility
+* `description`: multi-line description of the demo card
+* `hidden`: whether this card is shown in the generated index page
+
+# Configuration
+
+You can pass additional information by adding a YAML front matter to the markdown file.
+Supported items are:
+
+* `cover`: an URL or a relative path to the cover image. If not specified, it will use the first available image link, or all-white image if there's no image links.
+* `description`: a multi-line description to this file, will be displayed when the demo card is hovered. By default it uses `title`.
+* `id`: specify the `id` tag for cross-references. By default it's infered from the filename, e.g., `simple_demo` from `simple demo.md`.
+* `title`: one-line description to this file, will be displayed under the cover image. By default, it's the name of the file (without extension).
+* `author`: author name. If there are multiple authors, split them with semicolon `;`.
+* `julia`: Julia version compatibility. Any string that can be converted to `VersionNumber`
+* `date`: any string contents that can be passed to `Dates.DateTime`. For example, `2020-09-13`.
+* `hidden`: whether this card is shown in the layout of index page. The default value is `false`.
+
+An example of the front matter:
+
+```text
+---
+title: passing extra information
+cover: cover.png
+id: non_ambiguious_id
+author: Jane Doe; John Roe
+date: 2020-01-31
+description: this demo shows how you can pass extra demo information to DemoCards package. All these are optional.
+hidden: false
+---
+```
+
+See also: [`PlutoDemoCard`](@ref DemoCards.PlutoDemoCard), [`DemoSection`](@ref DemoCards.DemoSection), [`DemoPage`](@ref DemoCards.DemoPage)
+"""
+mutable struct PlutoDemoCard <: AbstractDemoCard
+ path::String
+ cover::Union{String, Nothing}
+ id::String
+ title::String
+ description::String
+ author::String
+ date::DateTime
+ julia::Union{Nothing, VersionNumber}
+ hidden::Bool
+end
+
+function PlutoDemoCard(path::AbstractString)::PlutoDemoCard
+ # first consturct an incomplete democard, and then load the config
+ card = PlutoDemoCard(path, "", "", "", "", "", DateTime(0), JULIA_COMPAT, false)
+
+ config = parse(card)
+ card.cover = load_config(card, "cover"; config=config)
+ card.title = load_config(card, "title"; config=config)
+ card.date = load_config(card, "date"; config=config)
+ card.author = load_config(card, "author"; config=config)
+ card.julia = load_config(card, "julia"; config=config)
+ # default id requires a title
+ card.id = load_config(card, "id"; config=config)
+ # default description requires a title
+ card.description = load_config(card, "description"; config=config)
+ card.hidden = load_config(card, "hidden"; config=config)
+
+ return card
+end
+
+
+"""
+ save_democards(card_dir::AbstractString, card::PlutoDemoCard;
+ project_dir,
+ src,
+ credit,
+ nbviewer_root_url)
+
+process the original julia file and save it.
+
+The processing pipeline is:
+
+1. preprocess and copy source file
+3. generate markdown file
+4. insert header and footer to generated markdown file
+"""
+function save_democards(card_dir::AbstractString,
+ card::PlutoDemoCard;
+ credit,
+ nbviewer_root_url,
+ project_dir=Base.source_dir(),
+ src="src",
+ throw_error = false,
+ properties = Dict{String, Any}(),
+ kwargs...)
+ if !isabspath(card_dir)
+ card_dir = abspath(card_dir)
+ end
+ isdir(card_dir) || mkpath(card_dir)
+ @debug card.path
+
+ # copy to card dir and do things
+ cardname = splitext(basename(card.path))[1]
+ curr_dir = pwd()
+ src_dir = dirname(card.path)
+ # pluto outputs are expensive, we save the output to a cache dir
+ # these cache dir contains the render files from previous runs,
+ # saves time, while rendering
+ render_dir = joinpath(src_dir, "..", "..", "pluto_output") |> abspath
+ isdir(render_dir) || mkpath(render_dir)
+
+ nb_path = joinpath(card_dir, "$(cardname).jl")
+ card_path = joinpath(card_dir, "$(cardname).md")
+
+ _, _, body = split_pluto_frontmatter(readlines(card.path))
+ write(nb_path, join(body, "\n"))
+
+ if VERSION < card.julia
+ # It may work, it may not work; I hope it would work.
+ @warn "The running Julia version `$(VERSION)` is older than the declared compatible version `$(card.julia)`. You might need to upgrade your Julia."
+ end
+
+ oopts = OutputOptions(; append_build_context=false)
+ output_format = documenter_output
+ bopts = BuildOptions(card_dir;previous_dir=render_dir,
+ output_format=output_format)
+ # don't run notebooks in parallel
+ build_notebooks(bopts, ["$(cardname).jl"], oopts)
+
+ cache_path = joinpath(render_dir, basename(card_path))
+ cp(card_path, cache_path; force=true)
+
+ badges = make_badges(card;
+ src=src,
+ card_dir=card_dir,
+ nbviewer_root_url=nbviewer_root_url,
+ project_dir=project_dir,
+ build_notebook=false) 
+
+ header = "# [$(card.title)](@id $(card.id))\n"
+ footer = pluto_footer
+
+ body = join(readlines(card_path), "\n")
+ write(card_path, header, badges * "\n\n", body, footer)
+
+ return nothing
+end
+
+function make_badges(card::PlutoDemoCard; src, card_dir, nbviewer_root_url, project_dir, build_notebook)
+ cardname = splitext(basename(card.path))[1]
+ badges = []
+ push!(badges, "[![Source code]($download_badge)]($(cardname).jl)")
+
+ push!(badges, invoke(make_badges, Tuple{AbstractDemoCard}, card))
+
+ join(badges, " ")
+end

src/utils.jl

@@ -143,6 +143,28 @@ function get_regex(::Val{:Markdown}, regex_type)
  end
 end
 
+
+function get_regex(::Val{:Pluto}, regex_type)
+ if regex_type == :image
+ # Example: ![title](path)
+ return r"^\s*!\[(?<title>[^\]]*)\]\((?<path>[^\s]*)\)"
+ elseif regex_type == :title
+ # Example: # [title](@id id)
+ regex_title = r"md[\"]{1,3}.*\n#\s(\S.*)\n"
+ # Example: # title
+ regex_simple_title = r"md[\"]{1,3}.*\n#\s(\S.*)\n"
+
+ # Note: return the complete one first
+ return (regex_title, regex_simple_title)
+ elseif regex_type == :content
+ # lines that are not title, image, link, list
+ # FIXME: list is also captured by this regex
+ return r"^\s*(?<content>[^#\-*!].*)"
+ else
+ error("Unrecognized regex type: $(regex_type)")
+ end
+end
+
 function get_regex(::Val{:Julia}, regex_type)
  if regex_type == :image
  # Example: #md ![title](path)
@@ -189,8 +211,14 @@ Currently supported items are: `title`, `id`, `cover`, `description`.
  They also need to validate the values.
 """
 function parse(T::Val, card::AbstractDemoCard)
- header, frontmatter, body = split_frontmatter(readlines(card.path))
- config = parse(T, body)
+ # TODO: generalize
+ if T === Val(:Pluto)
+ header, frontmatter, body = split_pluto_frontmatter(readlines(card.path))
+ config = parse(T, body) # should ideally not pickup
+ else
+ header, frontmatter, body = split_frontmatter(readlines(card.path))
+ config = parse(T, body)
+ end
  # frontmatter has higher priority
  if !isempty(frontmatter)
  yaml_config = try
@@ -207,6 +235,7 @@ function parse(T::Val, card::AbstractDemoCard)
 end
 parse(card::JuliaDemoCard) = parse(Val(:Julia), card)
 parse(card::MarkdownDemoCard) = parse(Val(:Markdown), card)
+parse(card::PlutoDemoCard) = parse(Val(:Pluto), card)
 
 
 function parse(T::Val, contents::String)
@@ -313,7 +342,7 @@ function split_frontmatter(contents::AbstractArray{<:AbstractString})
  m isa RegexMatch
  end
  offsets = findall(offsets)
- 
+
  length(offsets) < 2 && return "", "", contents
  if offsets[1] != 1 && !all(x->isempty(strip(x)) || startswith(strip(x), "#"), contents[1:offsets[1]-1])
  # For julia demo, comments and empty lines are allowed before frontmatter
@@ -341,6 +370,28 @@ function split_frontmatter(contents::AbstractArray{<:AbstractString})
 end
 
 
+# special case, generalize it with rest of functions later
+function split_pluto_frontmatter(contents)
+ first_cell_regex = r"#\s╔═╡\sCell\sorder:\n#\s?[╠╟][─═]\s?([0-9a-f\-]{36})"
+ content = join(contents, "\n")
+ first_cell_id = match(first_cell_regex, content)[1]
+ m1 = r"#\s╔═╡\s"
+ m2 = Regex("$(first_cell_id)\n")
+ m3 = r"""md\"\"\"\n([\s\S]*?)\"\"\""""
+ first_cell_regex = m1 * m2 * m3
+ frontmatter = match(first_cell_regex, content)[1]
+ frontmatter = split(frontmatter, "\n") |> Vector{String}
+ content_id_repr = r"#\s?[╠╟][─═]\s?" * Regex("$(first_cell_id)")
+
+ offset = map(contents) do line
+ m = match(content_id_repr, line)
+ m isa RegexMatch
+ end |> findlast
+
+ return String[], frontmatter, vcat(contents[1:offset-1], contents[offset+1:end])
+end
+
+
 function get_default_title(x::Union{AbstractDemoCard, DemoSection, DemoPage})
  name_without_ext = splitext(basename(x))[1]
  strip(replace(uppercasefirst(name_without_ext), r"[_-]" => " "))
@@ -369,3 +420,5 @@ function input_bool(prompt)
  # Otherwise loop and repeat the prompt
  end
 end
+
+is_pluto_notebook(path::String) = any(occursin.(r"#\s?╔═╡\s?[0-9a-f\-]{36}", readlines(path)))