Add tests for failures

Co-authored-by: Jules Aguillon <juloo.dsi@gmail.com>

test/bin/dune

@@ -1,5 +1,4 @@
 (executable
  (name gen_dune_rules)
  (libraries cmdliner fmt unix)
- (modules gen_dune_rules)
-)
+ (modules gen_dune_rules))

test/bin/gen_dune_rules.ml

@@ -30,6 +30,12 @@ let read_file filename =
     close_in chan;
     List.rev !lines
 
+let expected_file file =
+  let e = file ^ ".expected" in
+  if not (Sys.file_exists e) then
+    Fmt.failwith "No %s file found when generating an expect rule" e;
+  e
+
 (* Copied from Filename (stdlib) for pre-4.04 compatibility *)
 let chop_extension name =
   let is_dir_sep s i = match Sys.os_type with
@@ -71,11 +77,7 @@ let test_fixpoint_rule file =
 (* Tests that the output of 'ocaml-mdx test [options] <file>' is equal to the
    contents of '<file>.expected'. *)
 let test_expect_rule file =
-  let expected_file = file ^ ".expected" in
-  if not (Sys.file_exists expected_file) then
-    failwith (
-      Fmt.strf "No %s file found when generating an expect rule" expected_file
-    );
+  let expected_file = expected_file file in
   let options = options_of_file file in
   Fmt.pr
     {|
@@ -93,6 +95,27 @@ let test_expect_rule file =
     expected_file
     pp_options options
 
+let test_failure_rule file =
+  let expected_file = expected_file file in
+  let options = options_of_file file in
+  let target_file = file ^ ".actual" in
+  Fmt.pr
+    {|
+(rule
+  (targets %s)
+  (action
+    (with-outputs-to %%{targets}
+      (system "! %%{bin:ocaml-mdx} test %a %%{dep:%s}"))))
+
+(alias
+  (name runtest)
+  (action
+    (diff %s %s)))
+|}
+    target_file
+    pp_options options file
+    expected_file target_file
+
 let rule_gen rule_type () =
   Sys.readdir "."
   |> Array.to_list
@@ -101,14 +124,16 @@ let rule_gen rule_type () =
   |> List.iter (match rule_type with
       | `Test_fixpoint -> test_fixpoint_rule
       | `Test_expect -> test_expect_rule
+      | `Test_failure -> test_failure_rule
     )
 
 open Cmdliner
 
 let cmds =
   Term.[
     const (rule_gen `Test_fixpoint) $ const (), info "test_fixpoint";
-    const (rule_gen `Test_expect) $ const (), info "test_expect"
+    const (rule_gen `Test_expect) $ const (), info "test_expect";
+    const (rule_gen `Test_failure) $ const (), info "test_failure";
   ]
 
 let default =

test/bin/test_failure/both_prelude.md.expected

@@ -0,0 +1 @@
+only one of --prelude or --prelude-str shoud be used

test/bin/test_failure/both_prelude.opts

@@ -0,0 +1 @@
+--prelude=a --prelude-str=b

test/bin/test_failure/dune

@@ -0,0 +1,12 @@
+(include dune.inc)
+
+(rule
+ (targets dune.gen)
+ (deps (source_tree .))
+ (action
+  (with-stdout-to %{targets}
+   (run ../gen_dune_rules.exe test_failure))))
+
+(alias
+ (name runtest)
+ (action (diff dune.inc dune.gen)))

test/bin/test_failure/dune.inc

@@ -0,0 +1,44 @@
+
+(rule
+  (targets both_prelude.md.actual)
+  (action
+    (with-outputs-to %{targets}
+      (system "! %{bin:ocaml-mdx} test --prelude=a --prelude-str=b %{dep:both_prelude.md}"))))
+
+(alias
+  (name runtest)
+  (action
+    (diff both_prelude.md.expected both_prelude.md.actual)))
+
+(rule
+  (targets in_toplevel.md.actual)
+  (action
+    (with-outputs-to %{targets}
+      (system "! %{bin:ocaml-mdx} test  %{dep:in_toplevel.md}"))))
+
+(alias
+  (name runtest)
+  (action
+    (diff in_toplevel.md.expected in_toplevel.md.actual)))
+
+(rule
+  (targets ml_file_not_found.md.actual)
+  (action
+    (with-outputs-to %{targets}
+      (system "! %{bin:ocaml-mdx} test  %{dep:ml_file_not_found.md}"))))
+
+(alias
+  (name runtest)
+  (action
+    (diff ml_file_not_found.md.expected ml_file_not_found.md.actual)))
+
+(rule
+  (targets part_not_found.md.actual)
+  (action
+    (with-outputs-to %{targets}
+      (system "! %{bin:ocaml-mdx} test --direction=to-md %{dep:part_not_found.md}"))))
+
+(alias
+  (name runtest)
+  (action
+    (diff part_not_found.md.expected part_not_found.md.actual)))

test/bin/test_failure/in_toplevel.md.expected

@@ -0,0 +1,2 @@
+Error in the toplevel code block in in_toplevel.md at line 4:
+./not_found.ml: No such file or directory

test/bin/test_failure/ml_file_not_found.md.expected

@@ -0,0 +1,2 @@
+Error in the OCaml code block in ml_file_not_found.md at line 4:
+./not_found.ml: No such file or directory

test/bin/test_failure/part_not_found.md

@@ -0,0 +1,4 @@
+Run this with `--direction=to-md`
+
+```ocaml file=part_not_found.ml,part=part1
+```

test/bin/test_failure/part_not_found.md.expected

@@ -0,0 +1,2 @@
+Error in the OCaml code block in part_not_found.md at line 4:
+Cannot find part "part1" in ./part_not_found.ml

test/bin/test_failure/part_not_found.opts

@@ -0,0 +1 @@
+--direction=to-md

test/bin/test_fixpoint/dune.inc

@@ -132,6 +132,14 @@
   (progn
    (run ocaml-mdx test --direction=to-md %{x})
    (diff? %{x} %{x}.corrected))))
+(alias
+ (name runtest)
+ (deps
+  (:x module_alias.md))
+ (action
+  (progn
+   (run ocaml-mdx test --direction=to-md %{x})
+   (diff? %{x} %{x}.corrected))))
 (alias
  (name runtest)
  (deps