feat: execute markdown with jupytext frontmatter (#216)

demo/docs/variational-inference-nb.ipynb

@@ -250,7 +250,7 @@
     "\n",
     "<details class=\"tip\">\n",
     "    <summary>Extra: Proof of transformation equation</summary>\n",
-    "    <p><br>To simplify notations, let's use $Y=T(X)$ instead of $\\zeta=T(\\theta)$. After reaching the results, we will put the values back. Also, let's denote cummulative distribution function (cdf) as $F$. There are two cases which respect to properties of function $T$.<br><br><strong>Case 1</strong> - When $T$ is an increasing function $$F_Y(y) = P(Y <= y) = P(T(X) <= y)\\\\\n",
+    "    <p><br>To simplify notations, let's use $Y=T(X)$ instead of $\\zeta=T(\\theta)$. After reaching the results, we will put the values back. Also, let's denote cumulative distribution function (cdf) as $F$. There are two cases which respect to properties of function $T$.<br><br><strong>Case 1</strong> - When $T$ is an increasing function $$F_Y(y) = P(Y <= y) = P(T(X) <= y)\\\\\n",
     "    = P\\left(X <= T^{-1}(y) \\right) = F_X\\left(T^{-1}(y) \\right)\\\\\n",
     "    F_Y(y) = F_X\\left(T^{-1}(y) \\right)$$Let's differentiate with respect to $y$ both sides - $$\\frac{\\mathrm{d} (F_Y(y))}{\\mathrm{d} y} = \\frac{\\mathrm{d} (F_X\\left(T^{-1}(y) \\right))}{\\mathrm{d} y}\\\\\n",
     "    P_Y(y) = P_X\\left(T^{-1}(y) \\right) \\frac{\\mathrm{d} (T^{-1}(y))}{\\mathrm{d} y}$$<strong>Case 2</strong> - When $T$ is a decreasing function $$F_Y(y) = P(Y <= y) = P(T(X) <= y) = P\\left(X >= T^{-1}(y) \\right)\\\\\n",

src/mkdocs_jupyter/nbconvert2.py

@@ -127,7 +127,7 @@ def custom_clean_html(element):
 
     _, extension = os.path.splitext(nb_path)
 
-    if extension == ".py":
+    if extension in (".py", ".md"):
         nb = jupytext.read(nb_path)
         nb_file = io.StringIO(jupytext.writes(nb, fmt="ipynb"))
         content, resources = exporter.from_file(
@@ -196,7 +196,7 @@ def nb2md(nb_path, start=0, end=None, execute=False, kernel_name=""):
 
     _, extension = os.path.splitext(nb_path)
 
-    if extension == ".py":
+    if extension in (".py", ".md"):
         nb = jupytext.read(nb_path)
         nb_file = io.StringIO(jupytext.writes(nb, fmt="ipynb"))
         body, resources = exporter.from_file(nb_file)

src/mkdocs_jupyter/plugin.py

@@ -1,9 +1,10 @@
 import os
 import pathlib
 
+import jupytext
 import markdown
-from markdown.extensions.toc import TocExtension
 import mkdocs
+from markdown.extensions.toc import TocExtension
 from mkdocs.config import config_options
 from mkdocs.structure.files import File, Files
 from mkdocs.structure.pages import Page
@@ -21,9 +22,7 @@ class NotebookFile(File):
     def __init__(self, file, use_directory_urls, site_dir, **kwargs):
         self.file = file
         self.dest_path = self._get_dest_path(use_directory_urls)
-        self.abs_dest_path = os.path.normpath(
-            os.path.join(site_dir, self.dest_path)
-        )
+        self.abs_dest_path = os.path.normpath(os.path.join(site_dir, self.dest_path))
         self.url = self._get_url(use_directory_urls)
 
     def __getattr__(self, item):
@@ -35,7 +34,7 @@ def is_documentation_page(self):
 
 class Plugin(mkdocs.plugins.BasePlugin):
     config_scheme = (
-        ("include", config_options.Type(list, default=["*.py", "*.ipynb"])),
+        ("include", config_options.Type(list, default=["*.py", "*.ipynb", "*.md"])),
         ("ignore", config_options.Type(list, default=[])),
         ("execute", config_options.Type(bool, default=False)),
         ("execute_ignore", config_options.Type(list, default=[])),
@@ -51,14 +50,26 @@ class Plugin(mkdocs.plugins.BasePlugin):
         ("include_requirejs", config_options.Type(bool, default=False)),
         ("toc_depth", config_options.Type(int, default=6)),
         ("data_files", config_options.Type(dict, default={})),
-
     )
-    _supported_extensions = [".ipynb", ".py"]
+    _supported_extensions = [".ipynb", ".py", ".md"]
 
     def should_include(self, file):
         ext = os.path.splitext(str(file.abs_src_path))[-1]
         if ext not in self._supported_extensions:
             return False
+        if ext == ".md":
+            # only include markdown files with jupytext frontmatter
+            # that explicitly specifies a python kernel
+            try:
+                data = jupytext.read(file.abs_src_path)
+                if not (
+                    (meta := data.get("metadata", {}))
+                    and (kernelspec := meta.get("kernelspec"))
+                    and kernelspec["language"] == "python"
+                ):
+                    return False
+            except Exception as e:
+                return False
         srcpath = pathlib.PurePath(file.abs_src_path)
         include = None
         ignore = None
@@ -73,9 +84,7 @@ def should_include(self, file):
     def on_files(self, files, config):
         ret = Files(
             [
-                NotebookFile(file, **config)
-                if self.should_include(file)
-                else file
+                NotebookFile(file, **config) if self.should_include(file) else file
                 for file in files
             ]
         )
@@ -95,14 +104,11 @@ def on_pre_page(self, page, config, files):
             include_requirejs = self.config["include_requirejs"]
             toc_depth = self.config["toc_depth"]
 
-            if (
-                self.config["execute_ignore"]
-                and len(self.config["execute_ignore"]) > 0
-            ):
+            if self.config["execute_ignore"] and len(self.config["execute_ignore"]) > 0:
                 for ignore_pattern in self.config["execute_ignore"]:
-                    ignore_this = pathlib.PurePath(
-                        page.file.abs_src_path
-                    ).match(ignore_pattern)
+                    ignore_this = pathlib.PurePath(page.file.abs_src_path).match(
+                        ignore_pattern
+                    )
                     if ignore_this:
                         exec_nb = False
 
@@ -155,20 +161,23 @@ def on_post_page(self, output_content, page, config):
             os.makedirs(nb_target_dir, exist_ok=True)
             copyfile(nb_source, nb_target)
             print(f"Copied jupyter file: {nb_source} to {nb_target}")
-        
+
         # Include data files
         data_files = self.config["data_files"].get(page.file.src_path, [])
         if data_files:
             for data_file in data_files:
                 data_source = data_file
                 data_source_name = os.path.basename(data_file)
-                data_target_dir = os.path.dirname(os.path.join(nb_target_dir, data_source))
+                data_target_dir = os.path.dirname(
+                    os.path.join(nb_target_dir, data_source)
+                )
                 data_target = os.path.join(data_target_dir, data_source_name)
-                
+
                 os.makedirs(data_target_dir, exist_ok=True)
                 copyfile(data_source, data_target)
             print(page.data_files)
 
+
 def _get_markdown_toc(markdown_source, toc_depth):
     md = markdown.Markdown(extensions=[TocExtension(toc_depth=toc_depth)])
     md.convert(markdown_source)

src/mkdocs_jupyter/tests/mkdocs/base-with-mds.yml

@@ -0,0 +1,17 @@
+site_name: site-with-python-files
+site_description: mkdocs-jupyter test site
+
+nav:
+  - Home: index.md
+  - Demo (md): demo-jupytext.md
+  - Equations (md): variational-inference.md
+
+plugins:
+  - mkdocs-jupyter
+
+markdown_extensions:
+  - codehilite:
+      guess_lang: false
+  - pymdownx.highlight:
+      use_pygments: true
+  - pymdownx.arithmatex

src/mkdocs_jupyter/tests/mkdocs/docs/demo-jupytext.md

@@ -0,0 +1,187 @@
+---
+jupyter:
+  jupytext:
+    text_representation:
+      extension: .md
+      format_name: markdown
+      format_version: '1.3'
+      jupytext_version: 1.16.4
+  kernelspec:
+    display_name: Python 3
+    language: python
+    name: python3
+---
+
+<!-- #region -->
+# Demo notebook
+
+## Header 2
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur purus mi, sollicitudin ac justo a, dapibus ultrices dolor. Curabitur id eros mattis, tincidunt ligula at, condimentum urna.
+
+### Header 3
+
+A regular markdown code block
+
+```python
+id_ = 0
+for directory in directories:
+    rootdir = os.path.join('/Users/drodriguez/Downloads/aclImdb', directory)
+    for subdir, dirs, files in os.walk(rootdir):
+        for file_ in files:
+            with open(os.path.join(subdir, file_), 'r') as f:
+                doc_id = '_*%i' % id_
+                id_ = id_ + 1
+
+                text = f.read()
+                text = text.decode('utf-8')
+                tokens = nltk.word_tokenize(text)
+                doc = ' '.join(tokens).lower()
+                doc = doc.encode('ascii', 'ignore')
+                input_file.write('%s %s\n' % (doc_id, doc))
+```
+
+### More markdown things
+
+> Pellentesque pretium euismod laoreet. Nullam eget mauris ut tellus vehicula consequat. In sed molestie metus. Nulla at various nunc, sit amet semper arcu. Integer tristique augue eget auctor aliquam. Donec ornare consectetur lectus et viverra. Duis vel elit ac lectus accumsan gravida non ac erat.
+
+Ut in ipsum id neque pellentesque iaculis. Pellentesque massa erat, rhoncus id auctor vel, tempor id neque. Nunc nec iaculis enim. Duis eget tincidunt tellus. Proin vitae ultrices velit.
+
+1. Item 1
+2. Curabitur vel enim at mi dictum venenatis eget eu nulla. Suspendisse potenti. Etiam vitae nibh a odio dictum aliquam. Sed sit amet adipiscing leo, vitae euismod arcu.
+3. Item 3
+
+Sed vestibulum justo et turpis ullamcorper, a interdum sapien tristique. Donec ullamcorper ipsum ac scelerisque lacinia. Quisque et eleifend odio. Curabitur vel enim at mi dictum venenatis eget eu nulla. Suspendisse potenti. Etiam vitae nibh a odio dictum aliquam. Sed sit amet adipiscing leo, vitae euismod arcu.
+
+- Item 1
+- Curabitur vel enim at mi dictum venenatis eget eu nulla. Suspendisse potenti. Etiam vitae nibh a odio dictum aliquam. Sed sit amet adipiscing leo, vitae euismod arcu.
+- Item 3
+
+![Alt text](http://img3.wikia.nocookie.net/__cb20130524024810/logopedia/images/f/fa/Apple_logo_black.svg "Image")
+
+<hr>
+
+Sed vestibulum justo et turpis ullamcorper, a interdum sapien tristique. Donec ullamcorper ipsum ac scelerisque lacinia. Quisque et eleifend odio. Curabitur vel enim at mi dictum venenatis eget eu nulla. Suspendisse potenti. Etiam vitae nibh a odio dictum aliquam. Sed sit amet adipiscing leo, vitae euismod arcu.
+<!-- #endregion -->
+
+## Code cells
+
+This first code cells have some tags
+
+```python tags=["tag1"]
+a = 1
+```
+
+```python tags=["tag1", "tag2"]
+a
+```
+
+```python tags=["tag1", "tag2", "tag3"]
+b = "pew"
+```
+
+```python
+b
+```
+
+```python
+import re
+```
+
+```python
+text = "foo bar\t baz \tqux"
+```
+
+```python
+re.split("\s+", text)
+```
+
+```latex
+\begin{align}
+\nabla \times \vec{\mathbf{B}} -\, \frac1c\, \frac{\partial\vec{\mathbf{E}}}{\partial t} & = \frac{4\pi}{c}\vec{\mathbf{j}} \\
+\nabla \cdot \vec{\mathbf{E}} & = 4 \pi \rho \\
+\nabla \times \vec{\mathbf{E}}\, +\, \frac1c\, \frac{\partial\vec{\mathbf{B}}}{\partial t} & = \vec{\mathbf{0}} \\
+\nabla \cdot \vec{\mathbf{B}} & = 0
+\end{align}
+```
+
+```python
+import numpy as np
+import pandas as pd
+```
+
+```python
+dates = pd.date_range("20130101", periods=6)
+df = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list("ABCD"))
+df
+```
+
+```python
+%matplotlib inline
+```
+
+```python
+import matplotlib.pyplot as plt
+```
+
+```python
+from pylab import *
+```
+
+```python
+x = linspace(0, 5, 10)
+y = x**2
+```
+
+```python
+figure()
+plot(x, y, "r")
+xlabel("x")
+ylabel("y")
+title("title")
+show()
+```
+
+```python
+num_points = 130
+y = np.random.random(num_points)
+plt.plot(y)
+```
+
+This is some text, here comes some latex
+
+
+## Javascript plots
+
+
+### plotly
+
+```python
+import plotly.express as px
+```
+
+```python
+df = px.data.iris()
+fig = px.scatter(df, x="sepal_width", y="sepal_length")
+fig.show()
+```
+
+### bokeh
+
+```python
+from bokeh.plotting import figure, output_notebook, show
+```
+
+```python
+output_notebook()
+```
+
+```python
+p = figure()
+p.line([1, 2, 3, 4, 5], [6, 7, 2, 4, 5], line_width=2)
+show(p)
+```
+
+```python
+
+```

src/mkdocs_jupyter/tests/mkdocs/docs/variational-inference.ipynb

@@ -250,7 +250,7 @@
     "\n",
     "<details class=\"tip\">\n",
     "    <summary>Extra: Proof of transformation equation</summary>\n",
-    "    <p><br>To simplify notations, let's use $Y=T(X)$ instead of $\\zeta=T(\\theta)$. After reaching the results, we will put the values back. Also, let's denote cummulative distribution function (cdf) as $F$. There are two cases which respect to properties of function $T$.<br><br><strong>Case 1</strong> - When $T$ is an increasing function $$F_Y(y) = P(Y <= y) = P(T(X) <= y)\\\\\n",
+    "    <p><br>To simplify notations, let's use $Y=T(X)$ instead of $\\zeta=T(\\theta)$. After reaching the results, we will put the values back. Also, let's denote cumulative distribution function (cdf) as $F$. There are two cases which respect to properties of function $T$.<br><br><strong>Case 1</strong> - When $T$ is an increasing function $$F_Y(y) = P(Y <= y) = P(T(X) <= y)\\\\\n",
     "    = P\\left(X <= T^{-1}(y) \\right) = F_X\\left(T^{-1}(y) \\right)\\\\\n",
     "    F_Y(y) = F_X\\left(T^{-1}(y) \\right)$$Let's differentiate with respect to $y$ both sides - $$\\frac{\\mathrm{d} (F_Y(y))}{\\mathrm{d} y} = \\frac{\\mathrm{d} (F_X\\left(T^{-1}(y) \\right))}{\\mathrm{d} y}\\\\\n",
     "    P_Y(y) = P_X\\left(T^{-1}(y) \\right) \\frac{\\mathrm{d} (T^{-1}(y))}{\\mathrm{d} y}$$<strong>Case 2</strong> - When $T$ is a decreasing function $$F_Y(y) = P(Y <= y) = P(T(X) <= y) = P\\left(X >= T^{-1}(y) \\right)\\\\\n",