From 1f8ddedbf6679e3ab19f5f137b072455afbdfc78 Mon Sep 17 00:00:00 2001 From: Anil Madhavapeddy Date: Fri, 20 Sep 2024 15:47:30 +0100 Subject: [PATCH] First cut of jupyter output Signed-off-by: Jon Ludlam Coauthored-by: Anil Madhavapeddy --- CHANGES.md | 2 + bin/dune | 14 ++- bin/jupyter.ml | 119 ++++++++++++++++++ bin/main.ml | 3 +- bin/notebook.atd | 57 +++++++++ dune-project | 2 + mdx.opam | 2 + test/bin/misc-test-cases/mdx-jupyter/dune | 17 +++ .../mdx-jupyter/test.jpynb.expected | 1 + test/bin/misc-test-cases/mdx-jupyter/test.md | 16 +++ 10 files changed, 231 insertions(+), 2 deletions(-) create mode 100644 bin/jupyter.ml create mode 100644 bin/notebook.atd create mode 100644 test/bin/misc-test-cases/mdx-jupyter/dune create mode 100644 test/bin/misc-test-cases/mdx-jupyter/test.jpynb.expected create mode 100644 test/bin/misc-test-cases/mdx-jupyter/test.md diff --git a/CHANGES.md b/CHANGES.md index 25bc97b4f..fe0f14479 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,7 @@ ### 2.4.1 +- Add Jupyter Notebook output (#124, @avsm, @CraigFe, @jonludlam) + #### Changed - Revert #446: "Allow execution of included OCaml code blocks" (#451, @gpetiot). diff --git a/bin/dune b/bin/dune index f3ce6e0c1..2f9c1f89b 100644 --- a/bin/dune +++ b/bin/dune @@ -1,7 +1,7 @@ (library (name cli) (modules cli) - (libraries cmdliner fmt.cli logs.fmt fmt.tty logs.cli mdx)) + (libraries cmdliner fmt.cli logs.fmt fmt.tty logs.cli mdx yojson atdgen)) (executable (name main) @@ -9,3 +9,15 @@ (package mdx) (modules :standard \ cli) (libraries cli mdx)) + +(rule + (targets notebook_j.ml notebook_j.mli) + (deps notebook.atd) + (action + (run atdgen -j -j-std %{deps}))) + +(rule + (targets notebook_t.ml notebook_t.mli) + (deps notebook.atd) + (action + (run atdgen -t %{deps}))) diff --git a/bin/jupyter.ml b/bin/jupyter.ml new file mode 100644 index 000000000..f7ac44355 --- /dev/null +++ b/bin/jupyter.ml @@ -0,0 +1,119 @@ +open Mdx.Util.Result.Infix +open Cmdliner + +let raw t = + Notebook_t. + { + cell_type = `Raw; + metadata = { collapsed = None; scrolled = None }; + source = String.concat "\n" t; + outputs = None; + execution_count = None; + } + +let txt source = + Notebook_t. + { + cell_type = `Markdown; + metadata = { collapsed = None; scrolled = None }; + source; + outputs = None; + execution_count = None; + } + +let execution_count = ref 1 + +let ocaml contents = + let cell = + Notebook_t. + { + cell_type = `Code; + metadata = { collapsed = None; scrolled = None }; + source = String.concat "\n" contents; + outputs = Some []; + execution_count = Some !execution_count; + } + in + incr execution_count; + cell + +let toplevel x = + let cell = + Notebook_t. + { + cell_type = `Code; + metadata = { collapsed = None; scrolled = None }; + source = String.concat "\n" x.Mdx.Toplevel.command; + outputs = Some []; + execution_count = Some !execution_count; + } + in + incr execution_count; + cell + +let metadata = + Notebook_t. + { + kernelspec = + { + display_name = "OCaml 4.07.1"; + language = "OCaml"; + name = "ocaml-jupyter"; + }; + language_info = + { + name = "OCaml"; + version = "4.07.1"; + codemirror_mode = Some "text/x-ocaml"; + file_extension = ".ml"; + mimetype = "text/x-ocaml"; + nbconverter_exporter = None; + pygments_lexer = "OCaml"; + }; + } + +let rec collapse_text = function + | Mdx.Text x :: Mdx.Text y :: xs -> + collapse_text (Mdx.Text (x ^ "\n" ^ y) :: xs) + | (Mdx.Section _ as s) :: Mdx.Text y :: xs -> + let s = Mdx.to_string [ s ] in + collapse_text (Mdx.Text (s ^ "\n" ^ y) :: xs) + | (Mdx.Section _ as s) :: xs -> + let s = Mdx.to_string [ s ] in + collapse_text (Mdx.Text s :: xs) + | x :: ys -> x :: collapse_text ys + | [] -> [] + +let run _setup (`File file) = + Mdx.run_to_stdout file ~f:(fun _file_contents items -> + let cells = + List.fold_left + (fun acc -> function + | Mdx.Text "" -> acc + | Mdx.Text x -> txt x :: acc + | Mdx.Block { value = OCaml { env = User_defined _; _ }; _ } + | Mdx.Block { value = Toplevel { env = User_defined _; _ }; _ } -> + failwith + "internal error, cannot handle user defined environments" + | Mdx.Block { value = OCaml _; contents; _ } -> + ocaml contents :: acc + | Mdx.Block { value = Toplevel _; contents; loc; _ } -> + let blocks = Mdx.Toplevel.of_lines ~loc contents in + let newcells = List.rev_map toplevel blocks.Mdx.Toplevel.tests in + newcells @ acc + | Mdx.Block { value = Raw _; contents; _ } -> raw contents :: acc + | x -> + failwith + (Printf.sprintf "internal error, cannot handle: %s" + (Mdx.to_string [ x ]))) + [] (collapse_text items) + |> List.rev + in + Notebook_j.string_of_notebook + Notebook_t.{ metadata; nbformat = 4; nbformat_minor = 2; cells }) + >>! fun () -> 0 + +let term = Term.(const run $ Cli.setup $ Cli.file) +let doc = "Convert an mdx file to a jupyter notebook." +let info = Cmd.info "jupyter" ~doc +let cmd = Cmd.v info term diff --git a/bin/main.ml b/bin/main.ml index 7897cf14e..4edadf50c 100644 --- a/bin/main.ml +++ b/bin/main.ml @@ -16,7 +16,8 @@ open Cmdliner -let cmds = [ Test.cmd; Pp.cmd; Deps.cmd; Dune_gen.cmd ] +let cmds = [ Test.cmd; Pp.cmd; Deps.cmd; Dune_gen.cmd; Jupyter.cmd ] + let main (`Setup ()) = `Help (`Pager, None) let info = diff --git a/bin/notebook.atd b/bin/notebook.atd new file mode 100644 index 000000000..9d5e2381c --- /dev/null +++ b/bin/notebook.atd @@ -0,0 +1,57 @@ +type kernelspec = { + display_name : string; + language: string; + name: string; +} + +type language_info = { + name: string; + version: string; + codemirror_mode: string nullable; + file_extension: string; + mimetype: string; + nbconverter_exporter: string nullable; + pygments_lexer: string; +} + +type metadata = { + kernelspec: kernelspec; + language_info: language_info; +} + +type cell_metadata = { + ?collapsed: bool nullable; + ?scrolled: bool nullable; +} + +type cell_type = [ + | Code + | Markdown + | Raw +] + +type output_type = [ + Stream + | Display_data + | Execute_result + | Error +] + +type output = { + output_type : output_type; +} + +type cell = { + cell_type : cell_type; + metadata: cell_metadata; + source: string; + ?outputs: output list nullable; + ?execution_count: int option; +} + +type notebook = { + metadata: metadata; + nbformat: int; + nbformat_minor: int; + cells: cell list +} diff --git a/dune-project b/dune-project index c641a2dba..dbc2ec915 100644 --- a/dune-project +++ b/dune-project @@ -29,11 +29,13 @@ (csexp (>= 1.3.2)) astring + atdgen (logs (>= 0.7.0)) (cmdliner (>= 1.1.0)) (re (>= 1.7.2)) + yojson (ocaml-version (>= 2.3.0)) (lwt :with-test) diff --git a/mdx.opam b/mdx.opam index 833e964d8..d39642b97 100644 --- a/mdx.opam +++ b/mdx.opam @@ -25,9 +25,11 @@ depends: [ "cppo" {build & >= "1.1.0"} "csexp" {>= "1.3.2"} "astring" + "atdgen" "logs" {>= "0.7.0"} "cmdliner" {>= "1.1.0"} "re" {>= "1.7.2"} + "yojson" "ocaml-version" {>= "2.3.0"} "lwt" {with-test} "camlp-streams" diff --git a/test/bin/misc-test-cases/mdx-jupyter/dune b/test/bin/misc-test-cases/mdx-jupyter/dune new file mode 100644 index 000000000..165a7ba89 --- /dev/null +++ b/test/bin/misc-test-cases/mdx-jupyter/dune @@ -0,0 +1,17 @@ +(rule + (targets test.jpynb) + (deps + (:x test.md) + (package mdx)) + (action + (with-stdout-to + %{targets} + (run %{bin:ocaml-mdx} jupyter %{x})))) + +(rule + (alias runtest) + (deps + (:x test.jpynb) + (:y test.jpynb.expected)) + (action + (diff? %{y} %{x}))) diff --git a/test/bin/misc-test-cases/mdx-jupyter/test.jpynb.expected b/test/bin/misc-test-cases/mdx-jupyter/test.jpynb.expected new file mode 100644 index 000000000..99e9e333a --- /dev/null +++ b/test/bin/misc-test-cases/mdx-jupyter/test.jpynb.expected @@ -0,0 +1 @@ +{"metadata":{"kernelspec":{"display_name":"OCaml 4.07.1","language":"OCaml","name":"ocaml-jupyter"},"language_info":{"name":"OCaml","version":"4.07.1","codemirror_mode":"text/x-ocaml","file_extension":".ml","mimetype":"text/x-ocaml","nbconverter_exporter":null,"pygments_lexer":"OCaml"}},"nbformat":4,"nbformat_minor":2,"cells":[{"cell_type":"markdown","metadata":{},"source":"## Generating a Jupyter notebook\n\n\n\n"},{"cell_type":"code","metadata":{},"source":"\nlet x = 1\nlet () = assert true\n","outputs":[],"execution_count":1},{"cell_type":"markdown","metadata":{},"source":"\n\nThis block should not be executed by the tests:\n"},{"cell_type":"code","metadata":{},"source":"\nlet x = 2\nlet () = assert false (* Don't print this! *)\n","outputs":[],"execution_count":2},{"cell_type":"markdown","metadata":{},"source":"\n"},{"cell_type":"code","metadata":{},"source":"\nlet () = print_int x\n","outputs":[],"execution_count":3}]} \ No newline at end of file diff --git a/test/bin/misc-test-cases/mdx-jupyter/test.md b/test/bin/misc-test-cases/mdx-jupyter/test.md new file mode 100644 index 000000000..f32521f74 --- /dev/null +++ b/test/bin/misc-test-cases/mdx-jupyter/test.md @@ -0,0 +1,16 @@ +## Generating a Jupyter notebook + +```ocaml +let x = 1 +let () = assert true +``` + +This block should not be executed by the tests: +```ocaml skip +let x = 2 +let () = assert false (* Don't print this! *) +``` + +```ocaml +let () = print_int x +```