Introduce Markdown.AST

- Introduce ExDoc.Markdown.AST and t:Markdown.AST.html/0.
- Replaces ExDoc.Formatter.HTML.ast_to_html/1 with ExDoc.Markdown.AST.to_html/2 which is a port of Earmark.Transform
- It uses a more recent git version of Earkmark since due to a bug in the library.
- It adds Floki to deal with HTML code in Markdown docs.

Closes #1168, #1189
Supersedes #1190

lib/ex_doc/autolink.ex

@@ -48,7 +48,7 @@ defmodule ExDoc.Autolink do
 
   @autoimported_modules [Kernel, Kernel.SpecialForms]
 
-  def doc(ast, options \\ []) do
+  def doc(ast, options) do
     config = struct!(__MODULE__, options)
     walk(ast, config)
   end
@@ -61,34 +61,34 @@ defmodule ExDoc.Autolink do
     binary
   end
 
-  defp walk({:pre, _, _} = ast, _config) do
+  defp walk({:pre, _metadata, _, _} = ast, _config) do
     ast
   end
 
-  defp walk({:a, attrs, inner} = ast, config) do
+  defp walk({:a, metadata, attrs, inner} = ast, config) do
     cond do
       url = custom_link(attrs, config) ->
-        {:a, Keyword.put(attrs, :href, url), inner}
+        {:a, metadata, Keyword.put(attrs, :href, url), inner}
 
       url = extra_link(attrs, config) ->
-        {:a, Keyword.put(attrs, :href, url), inner}
+        {:a, metadata, Keyword.put(attrs, :href, url), inner}
 
       true ->
         ast
     end
   end
 
-  defp walk({:code, attrs, [code]} = ast, config) do
+  defp walk({:code, metadata, attrs, [code]} = ast, config) do
     if url = url(code, :regular, config) do
       code = remove_prefix(code)
-      {:a, [href: url], [{:code, attrs, [code]}]}
+      {:a, metadata, [href: url], [{:code, metadata, attrs, [code]}]}
     else
       ast
     end
   end
 
-  defp walk({tag, attrs, ast}, config) do
-    {tag, attrs, walk(ast, config)}
+  defp walk({tag, metadata, attrs, ast}, config) do
+    {tag, metadata, attrs, walk(ast, config)}
   end
 
   defp custom_link(attrs, config) do

lib/ex_doc/formatter/html.ex

@@ -1,6 +1,8 @@
 defmodule ExDoc.Formatter.HTML do
   @moduledoc false
 
+  alias ExDoc.Markdown.AST
+
   alias __MODULE__.{Assets, Templates, SearchItems}
   alias ExDoc.{Autolink, Markdown, GroupMatcher}
 
@@ -117,20 +119,10 @@ defmodule ExDoc.Formatter.HTML do
   defp autolink_and_render(doc, autolink_opts, opts) do
     doc
     |> Autolink.doc(autolink_opts)
-    |> ast_to_html()
-    |> IO.iodata_to_binary()
+    |> AST.to_html()
     |> ExDoc.Highlighter.highlight_code_blocks(opts)
   end
 
-  @doc false
-  def ast_to_html(list) when is_list(list), do: Enum.map(list, &ast_to_html/1)
-  def ast_to_html(binary) when is_binary(binary), do: Templates.h(binary)
-
-  def ast_to_html({tag, attrs, ast}) do
-    attrs = Enum.map(attrs, fn {key, val} -> " #{key}=\"#{val}\"" end)
-    ["<#{tag}", attrs, ">", ast_to_html(ast), "</#{tag}>"]
-  end
-
   defp output_setup(build, config) do
     if File.exists?(build) do
       build

lib/ex_doc/markdown/ast.ex

@@ -0,0 +1,161 @@
+defmodule ExDoc.Markdown.AST do
+  @type html :: [html_element()]
+
+  @type html_element :: {html_tag(), metadata(), html_attributes(), children()} | String.t()
+  @type html_tag :: atom()
+  @type metadata :: %{
+          optional(atom()) => any(),
+          optional(:line) => integer(),
+          optional(:column) => integer(),
+          optional(:verbatim) => boolean(),
+          optional(:comment) => boolean()
+        }
+  @type html_attributes :: Keyword.t(String.t())
+  @type children :: ast()
+
+  # https://www.w3.org/TR/2011/WD-html-markup-20110113/syntax.html#void-element
+  @void_elements ~W(area base br col command embed hr img input keygen link meta param source track wbr)a
+
+  # Ported from: Earmark.Transform
+  # Copyright (c) 2014 Dave Thomas, The Pragmatic Programmers @/+pragdave, dave@pragprog.com
+  # Apache License v2.0
+  # https://github.com/pragdave/earmark/blob/a2a85bc3f2e262a2c697ed8001b0eaa06ee42d92/lib/earmark/transform.ex
+  @spec to_html(html()) :: String.t()
+  def to_html(ast, options \\ %{})
+
+  def to_html(ast, options) do
+    ast
+    |> to_html(options, false)
+    |> IO.iodata_to_binary()
+  end
+
+  defp to_html(elements, options, verbatim) when is_list(elements) do
+    Enum.map(elements, &to_html(&1, options, verbatim))
+  end
+
+  defp to_html(element, options, false) when is_binary(element) do
+    case escape_with_options(element, options) do
+      "" ->
+        []
+
+      content ->
+        [content]
+    end
+  end
+
+  defp to_html(element, _options, true) when is_binary(element) do
+    [element]
+  end
+
+  # Void element
+  defp to_html({tag, _metadata, attributes, []}, _options, _verbatim) when tag in @void_elements,
+    do: open_element(tag, attributes)
+
+  # Comment
+  defp to_html({nil, %{comment: true}, _attributes, children}, _options, _verbatim) do
+    ["<!--", Enum.intersperse(children, ["\n"]), "-->"]
+  end
+
+  defp to_html({:code, _metadata, attributes, children}, _options, __verbatim) do
+    [
+      open_element(:code, attributes),
+      children |> Enum.join("\n") |> escape(true),
+      "</code>"
+    ]
+  end
+
+  defp to_html({:pre, metadata, attributes, children}, options, verbatim) do
+    verbatim_new = metadata[:verbatim] || verbatim
+
+    [
+      open_element(:pre, attributes),
+      to_html(Enum.intersperse(children, ["\n"]), options, verbatim_new),
+      "</pre>\n"
+    ]
+  end
+
+  # Element with no children
+  defp to_html({tag, _metadata, attributes, []}, _options, _verbatim) do
+    [open_element(tag, attributes), "</#{tag}>", "\n"]
+  end
+
+  # Element with children
+  defp to_html({tag, metadata, attributes, children}, options, verbatim) do
+    verbatim_new = metadata[:verbatim] || verbatim
+
+    [open_element(tag, attributes), to_html(children, options, verbatim_new), "</#{tag}>"]
+  end
+
+  defp make_attribute(name_value_pair, tag)
+
+  defp make_attribute({name, value}, _) do
+    [" ", "#{name}", "=\"", to_string(value), "\""]
+  end
+
+  defp open_element(tag, attributes) when tag in @void_elements do
+    ["<", "#{tag}", Enum.map(attributes, &make_attribute(&1, tag)), " />"]
+  end
+
+  defp open_element(tag, attributes) do
+    ["<", "#{tag}", Enum.map(attributes, &make_attribute(&1, tag)), ">"]
+  end
+
+  @em_dash_regex ~r{---}
+  @en_dash_regex ~r{--}
+  @dbl1_regex ~r{(^|[-–—/\(\[\{"”“\s])'}
+  @single_regex ~r{\'}
+  @dbl2_regex ~r{(^|[-–—/\(\[\{‘\s])\"}
+  @dbl3_regex ~r{"}
+  defp smartypants(text, options)
+
+  defp smartypants(text, %{smartypants: true}) do
+    text
+    |> replace(@em_dash_regex, "—")
+    |> replace(@en_dash_regex, "–")
+    |> replace(@dbl1_regex, "\\1‘")
+    |> replace(@single_regex, "’")
+    |> replace(@dbl2_regex, "\\1“")
+    |> replace(@dbl3_regex, "”")
+    |> String.replace("...", "…")
+  end
+
+  defp smartypants(text, _options), do: text
+
+  defp replace(text, regex, replacement, options \\ []) do
+    Regex.replace(regex, text, replacement, options)
+  end
+
+  # Originally taken from: Earmark.Helpers
+  # Copyright (c) 2014 Dave Thomas, The Pragmatic Programmers @/+pragdave, dave@pragprog.com
+  # Apache License v2.0
+  # https://github.com/pragdave/earmark/blob/a2a85bc3f2e262a2c697ed8001b0eaa06ee42d92/lib/earmark/helpers.ex
+  # 
+  # Replace <, >, and quotes with the corresponding entities. If
+  # `encode` is true, convert ampersands, too, otherwise only
+  #  convert non-entity ampersands.
+  def escape(html, encode \\ false)
+
+  def escape(html, false) when is_binary(html),
+    do: escape_replace(Regex.replace(~r{&(?!#?\w+;)}, html, "&amp;"))
+
+  def escape(html, _) when is_binary(html), do: escape_replace(String.replace(html, "&", "&amp;"))
+
+  defp escape_replace(html) do
+    html
+    |> String.replace("<", "&lt;")
+    |> String.replace(">", "&gt;")
+    |> String.replace("\"", "&quot;")
+    |> String.replace("'", "&#39;")
+  end
+
+  defp escape_with_options(element, options)
+
+  defp escape_with_options("", _options),
+    do: ""
+
+  defp escape_with_options(element, options) do
+    element
+    |> smartypants(options)
+    |> escape()
+  end
+end