🐛 FIX: Add lexer to source code

executablebooks · Aug 19, 2020 · 39c1bb9 · 39c1bb9
1 parent 48deccc
commit 39c1bb9
Show file tree

Hide file tree

Showing 24 changed files with 142 additions and 57 deletions.
diff --git a/docs/examples/coconut-lang.md b/docs/examples/coconut-lang.md
@@ -0,0 +1,48 @@
+---
+jupytext:
+  text_representation:
+    extension: .md
+    format_name: myst
+    format_version: '0.12'
+    jupytext_version: 1.5.2
+kernelspec:
+  display_name: Coconut
+  language: coconut
+  name: coconut
+---
+
+# Notebooks in other languages
+
+<http://coconut-lang.org/>
+
+```{code-cell} coconut
+def factorial(n):
+    """Compute n! where n is an integer >= 0."""
+    if n `isinstance` int and n >= 0:
+        acc = 1
+        for x in range(1, n+1):
+            acc *= x
+        return acc
+    else:
+        raise TypeError("the argument to factorial must be an integer >= 0")
+
+3 |> factorial |> print
+```
+
+```{code-cell} coconut
+def quick_sort(l):
+    """Sort the input iterator using the quick sort algorithm."""
+    match [head] :: tail in l:
+        tail = reiterable(tail)
+        yield from quick_sort(left) :: [head] :: quick_sort(right) where:
+            left = (x for x in tail if x < head)
+            right = (x for x in tail if x >= head)
+    # By yielding nothing if the match falls through, we implicitly return an empty iterator.
+
+[3,0,4,2,1] |> quick_sort |> list |> print
+```
+
+```{code-cell} coconut
+:tags: [raises-exception]
+x
+```
diff --git a/docs/examples/index.md b/docs/examples/index.md
@@ -5,6 +5,7 @@ and reference.
 
 ```{toctree}
 basic
+coconut-lang
 interactive
 jupyter_sphinx
 ```
diff --git a/myst_nb/parser.py b/myst_nb/parser.py
@@ -112,6 +112,11 @@ def parse_block(src, start_line):
     block_tokens = []
     source_map = ntbk.metadata.get("source_map", None)
 
+    # get language lexer name
+    langinfo = ntbk.metadata.get("language_info", {})
+    lexer = langinfo.get("pygments_lexer", langinfo.get("name", None))
+    # TODO log warning if lexer is still None
+
     for cell_index, nb_cell in enumerate(ntbk.cells):
 
         # if the the source_map has been stored (for text-based notebooks),
@@ -144,7 +149,7 @@ def parse_block(src, start_line):
                     "nb_code_cell",
                     "",
                     0,
-                    meta={"cell": nb_cell},
+                    meta={"cell": nb_cell, "lexer": lexer},
                     map=[start_line, start_line],
                 )
             )
@@ -209,7 +214,9 @@ def render_nb_code_cell(self, token: Token):
             sphinx_cell += cell_input
 
             # Input block
-            code_block = nodes.literal_block(text=cell["source"])
+            code_block = nodes.literal_block(
+                text=cell["source"], language=token.meta["lexer"]
+            )
             cell_input += code_block
 
         # ==================

diff --git a/setup.py b/setup.py
@@ -66,6 +66,7 @@
             "pandas",
         ],
         "rtd": [
+            "coconut~=1.4.3",
             "sphinxcontrib-bibtex",
             "ipywidgets",
             "pandas",

diff --git a/tests/nb_fixtures/basic.txt b/tests/nb_fixtures/basic.txt
@@ -28,11 +28,36 @@ cells:
     <docinfo>
     <CellNode cell_type="code" classes="cell">
         <CellInputNode classes="cell_input">
-            <literal_block xml:space="preserve">
+            <literal_block language="True" xml:space="preserve">
                 a=1
                 print(a)
 .
 
+Code Cell (with lexer):
+.
+metadata:
+    language_info:
+        pygments_lexer: mylexer
+cells:
+  - cell_type: code
+    metadata: {}
+    execution_count: null
+    source: a=1
+    outputs: []
+.
+<document source="notset">
+    <docinfo>
+        <field>
+            <field_name>
+                language_info
+            <field_body>
+                {"pygments_lexer": "mylexer"}
+    <CellNode cell_type="code" classes="cell">
+        <CellInputNode classes="cell_input">
+            <literal_block language="mylexer" xml:space="preserve">
+                a=1
+.
+
 Code Cell (simple output):
 .
 cells:
@@ -51,7 +76,7 @@ cells:
     <docinfo>
     <CellNode cell_type="code" classes="cell">
         <CellInputNode classes="cell_input">
-            <literal_block xml:space="preserve">
+            <literal_block language="True" xml:space="preserve">
                 a=1
                 print(a)
         <CellOutputNode classes="cell_output">
@@ -84,7 +109,7 @@ cells:
             A Title
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="True" xml:space="preserve">
                     a=1
                     print(a)
         <paragraph>

diff --git a/tests/test_execute/test_basic_failing.xml b/tests/test_execute/test_basic_failing.xml
@@ -6,5 +6,5 @@
             some text
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     raise Exception('oopsie!')
diff --git a/tests/test_execute/test_basic_unrun.xml b/tests/test_execute/test_basic_unrun.xml
@@ -6,7 +6,7 @@
             some text
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     a=1
                     print(a)
             <CellOutputNode classes="cell_output">

diff --git a/tests/test_execute/test_basic_unrun_nbclient.xml b/tests/test_execute/test_basic_unrun_nbclient.xml
@@ -6,7 +6,7 @@
             some text
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     a=1
                     print(a)
             <CellOutputNode classes="cell_output">

diff --git a/tests/test_execute/test_complex_outputs_unrun.xml b/tests/test_execute/test_complex_outputs_unrun.xml
@@ -1,7 +1,7 @@
 <document source="complex_outputs_unrun">
     <CellNode cell_type="code" classes="cell">
         <CellInputNode classes="cell_input">
-            <literal_block xml:space="preserve">
+            <literal_block language="ipython3" xml:space="preserve">
                 import matplotlib.pyplot as plt
                 import pandas as pd
                 pd.set_option('display.latex.repr', True)
@@ -89,7 +89,7 @@
             Text Output
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     print("""
                     This is some printed text,
                     with a nicely formatted output.
@@ -113,7 +113,7 @@
             The plotting code for a pandas Dataframe table (\cref{tbl:example}).
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     np.random.seed(0) 
                     df = pd.DataFrame(np.random.rand(3,4),columns=['a','b','c','d'])
                     df.a = ['$\delta$','x','y']
@@ -127,15 +127,15 @@
             Equations (with ipython or sympy)
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     Latex('$$ a = b+c $$')
             <CellOutputNode classes="cell_output">
                 <CellOutputBundleNode output_count="1">
         <paragraph>
             The plotting code for a sympy equation (=@eqn:example_sympy).
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     y = sym.Function('y')
                     n = sym.symbols(r'\alpha')
                     f = y(n)-2*y(n-1/sym.pi)-5*y(n-2)
@@ -150,7 +150,7 @@
                 ipywidgets
             <CellNode cell_type="code" classes="cell">
                 <CellInputNode classes="cell_input">
-                    <literal_block xml:space="preserve">
+                    <literal_block language="ipython3" xml:space="preserve">
                         import ipywidgets as widgets
                         widgets.Layout(model_id="1337h4x0R")
                 <CellOutputNode classes="cell_output">

diff --git a/tests/test_execute/test_complex_outputs_unrun_nbclient.xml b/tests/test_execute/test_complex_outputs_unrun_nbclient.xml
@@ -1,7 +1,7 @@
 <document source="complex_outputs_unrun">
     <CellNode cell_type="code" classes="cell">
         <CellInputNode classes="cell_input">
-            <literal_block xml:space="preserve">
+            <literal_block language="ipython3" xml:space="preserve">
                 import matplotlib.pyplot as plt
                 import pandas as pd
                 pd.set_option('display.latex.repr', True)
@@ -89,7 +89,7 @@
             Text Output
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     print("""
                     This is some printed text,
                     with a nicely formatted output.
@@ -113,7 +113,7 @@
             The plotting code for a pandas Dataframe table (\cref{tbl:example}).
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     np.random.seed(0) 
                     df = pd.DataFrame(np.random.rand(3,4),columns=['a','b','c','d'])
                     df.a = ['$\delta$','x','y']
@@ -127,15 +127,15 @@
             Equations (with ipython or sympy)
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     Latex('$$ a = b+c $$')
             <CellOutputNode classes="cell_output">
                 <CellOutputBundleNode output_count="1">
         <paragraph>
             The plotting code for a sympy equation (=@eqn:example_sympy).
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     y = sym.Function('y')
                     n = sym.symbols(r'\alpha')
                     f = y(n)-2*y(n-1/sym.pi)-5*y(n-2)
@@ -150,7 +150,7 @@
                 ipywidgets
             <CellNode cell_type="code" classes="cell">
                 <CellInputNode classes="cell_input">
-                    <literal_block xml:space="preserve">
+                    <literal_block language="ipython3" xml:space="preserve">
                         import ipywidgets as widgets
                         widgets.Layout(model_id="1337h4x0R")
                 <CellOutputNode classes="cell_output">

diff --git a/tests/test_execute/test_exclude_path.xml b/tests/test_execute/test_exclude_path.xml
@@ -6,6 +6,6 @@
             some text
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     a=1
                     print(a)
diff --git a/tests/test_execute/test_jupyter_cache_path.xml b/tests/test_execute/test_jupyter_cache_path.xml
@@ -6,7 +6,7 @@
             some text
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     a=1
                     print(a)
             <CellOutputNode classes="cell_output">

diff --git a/tests/test_execute/test_no_execute.xml b/tests/test_execute/test_no_execute.xml
@@ -6,6 +6,6 @@
             some text
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     a=1
                     print(a)
diff --git a/tests/test_execute/test_outputs_present.xml b/tests/test_execute/test_outputs_present.xml
@@ -6,7 +6,7 @@
             some text
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     a=1
                     print(a)
             <CellOutputNode classes="cell_output">

diff --git a/tests/test_glue/test_parser.xml b/tests/test_glue/test_parser.xml
@@ -4,11 +4,11 @@
             Glue Tests
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     from myst_nb import glue
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     glue("key_text1", "text1")
                     glue("key_float", 3.14159)
             <CellOutputNode classes="cell_output">
@@ -18,12 +18,12 @@
                     3.14159
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     glue("key_undisplayed", "undisplayed", display=False)
             <CellOutputNode classes="cell_output">
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     import pandas as pd
                     df = pd.DataFrame({"header": [1, 2, 3]})
                     glue("key_df", df)
@@ -68,7 +68,7 @@
                     </div>
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     import matplotlib.pyplot as plt
                     plt.plot([1, 2, 3])
                     glue("key_plt", plt.gcf(), display=False)
@@ -154,7 +154,7 @@
                 Math
             <CellNode cell_type="code" classes="cell">
                 <CellInputNode classes="cell_input">
-                    <literal_block xml:space="preserve">
+                    <literal_block language="ipython3" xml:space="preserve">
                         import sympy as sym
                         f = sym.Function('f')
                         y = sym.Function('y')

diff --git a/tests/test_parser/test_basic_run.xml b/tests/test_parser/test_basic_run.xml
@@ -6,7 +6,7 @@
             some text
         <CellNode cell_type="code" classes="cell">
             <CellInputNode classes="cell_input">
-                <literal_block xml:space="preserve">
+                <literal_block language="ipython3" xml:space="preserve">
                     a=1
                     print(a)
             <CellOutputNode classes="cell_output">