Test Stanza Proposal (#822)

Add tests and test stanza

These stanzas are used to easily define tests. If a test has a corresponding .expect file, it will be immediately considered as an expect test.

doc/dune-files.rst

@@ -240,6 +240,8 @@ Executables can also be linked as object or shared object files. See
         (section bin)
         (files (<name>.exe as <public-name>)))
 
+.. _shared-exe-fields:
+
 - ``(package <package>)`` if there is a ``(public_name ...)`` field, this
   specifies the package the executables are part of
 
@@ -548,6 +550,8 @@ The syntax is as follows:
 
 ``<name>`` is an alias name such as ``runtest``.
 
+.. _alias-fields:
+
 ``<deps-conf list>`` specifies the dependencies of the alias. See the
 `Dependency specification`_ section for more details.
 
@@ -698,6 +702,45 @@ With this dune file, running dune as follow will replace the
 
     $ dune build @runtest --auto-promote
 
+.. _tests-stanza:
+
+tests
+-----
+
+The ``tests`` stanza allows one to easily define multiple tests. For example we
+can define two tests at once with:
+
+.. code:: scheme
+
+   (tests
+    (names mytest expect_test)
+    <optional fields>)
+
+This will define an executable named ``mytest.exe`` that will be executed as
+part of the ``runtest`` alias. If the directory also contains an
+``expect_test.expected`` file, then ``expect_test`` will be used to define an
+expect test. That is, the test will be executed and its output will be compared
+to ``expect_test.expected``.
+
+The optional fields that are supported are a subset of the alias and executables
+fields. In particular, all fields except for ``public_names`` are supported from
+the `executables stanza <shared-exe-fields>`_. Alias fields apart from ``name``
+and ``action`` are allowed.
+
+test
+----
+
+The ``test`` stanza is the singular form of ``tests``. The only difference is
+that it's of the form:
+
+.. code:: scheme
+
+   (test
+    (name foo)
+    <optional fields>)
+
+where the ``name`` field is singular. The same optional fields are supported.
+
 .. _dune-env:
 
 env

doc/tests.rst

@@ -330,6 +330,14 @@ running your testsuite, simply add this to a jbuild file:
            ((name   runtest)
             (action (run ./tests.exe))))
 
+Hence to define an a test a pair of alias and executable stanzas are required.
+To simplify this common pattern, dune provides a :ref:`tests-stanza` stanza to
+define multiple tests and their aliases at once:
+
+.. code:: scheme
+
+   (tests (names test1 test2))
+
 Diffing the result
 ------------------
 
@@ -348,12 +356,21 @@ command. For instance let's consider this test:
            ((name runtest)
             (action (diff tests.expected test.output))))
 
-After having run ``tests.exe`` and dumping its output to
-``tests.output``, dune will compare the latter to
-``tests.expected``. In case of mismatch, dune will print a diff
-and then the ``dune promote`` command can be used to copy over the
-generated ``test.output`` file to ``tests.expected`` in the source
-tree. This provides a nice way of dealing with the usual *write code*,
+After having run ``tests.exe`` and dumping its output to ``tests.output``, dune
+will compare the latter to ``tests.expected``. In case of mismatch, dune will
+print a diff and then the ``dune promote`` command can be used to copy over the
+generated ``test.output`` file to ``tests.expected`` in the source tree.
+
+Alternatively, the :ref:`tests-stanza` also supports this style of tests.
+
+.. code:: scheme
+
+   (tests (names tests))
+
+Where dune expects a ``tests.expected`` file to exist to infer that this is an
+expect tests.
+
+This provides a nice way of dealing with the usual *write code*,
 *run*, *promote* cycle of testing. For instance:
 
 .. code:: bash

src/gen_rules.ml

@@ -953,6 +953,67 @@ module Gen(P : Install_rules.Params) = struct
            ~targets:Alias
            ~scope)
 
+  let tests_rules (t : Tests.t) ~dir ~scope ~all_modules ~modules_partitioner
+        ~dir_kind ~src_dir =
+    let test_kind (loc, name) =
+      let sources = SC.source_files sctx ~src_path:src_dir in
+      let expected_basename = name ^ ".expected" in
+      if String.Set.mem sources expected_basename then
+        `Expect
+          { Action.Unexpanded.Diff.
+            file1 = String_with_vars.make_text loc expected_basename
+          ; file2 = String_with_vars.make_text loc (name ^ ".output")
+          ; optional = false
+          ; mode = Text
+          }
+      else
+        `Regular
+    in
+    let regular_rule run_action alias loc =
+      { alias with Alias_conf.action = Some (loc, run_action) }
+    in
+    let expect_rule run_action (diff : Action.Unexpanded.Diff.t) alias loc =
+      let rule =
+        { Rule.
+          targets = Infer
+        ; deps = []
+        ; action =
+            (loc, Action.Unexpanded.Redirect (Stdout, diff.file2, run_action))
+        ; mode = Standard
+        ; locks = t.locks
+        ; loc
+        } in
+      let alias =
+        { alias with
+          Alias_conf.
+          action = Some (loc, Diff diff)
+        ; locks = t.locks
+        } in
+      (alias, rule)
+    in
+    List.iter t.exes.names ~f:(fun (loc, s) ->
+      let run_action =
+        Action.Unexpanded.Run
+          (String_with_vars.make_text loc ("./" ^ s ^ ".exe"), []) in
+      let base_alias =
+        { Alias_conf.
+          name = "runtest"
+        ; locks = []
+        ; package = t.package
+        ; deps = t.deps
+        ; action = None
+        } in
+      match test_kind (loc, s) with
+      | `Regular ->
+        alias_rules ~dir ~scope (regular_rule run_action base_alias loc)
+      | `Expect diff ->
+        let (alias, rule) =
+          expect_rule run_action diff base_alias loc in
+        alias_rules ~dir ~scope alias;
+        ignore (user_rule ~dir ~scope rule : Path.t list));
+    executables_rules t.exes ~dir ~all_modules ~scope ~dir_kind
+      ~modules_partitioner
+
   (* +-----------------------------------------------------------------+
      | Stanza                                                          |
      +-----------------------------------------------------------------+ *)
@@ -975,6 +1036,9 @@ module Gen(P : Install_rules.Params) = struct
         | Alias alias ->
           alias_rules alias ~dir ~scope;
           None
+        | Tests tests ->
+          Some (tests_rules tests ~dir ~scope ~all_modules ~src_dir
+                  ~modules_partitioner ~dir_kind:kind)
         | Copy_files { glob; _ } ->
           let src_dir =
             let loc = String_with_vars.loc glob in
@@ -1011,7 +1075,7 @@ module Gen(P : Install_rules.Params) = struct
       | _ -> ());
     Modules_partitioner.emit_errors modules_partitioner
 
-     let gen_rules ~dir components : Build_system.extra_sub_directories_to_keep =
+  let gen_rules ~dir components : Build_system.extra_sub_directories_to_keep =
     (match components with
      | ".js"  :: rest -> Js_of_ocaml_rules.setup_separate_compilation_rules
                            sctx rest

src/jbuild.ml

@@ -1352,6 +1352,43 @@ module Alias_conf = struct
          })
 end
 
+module Tests = struct
+  type t =
+    { exes    : Executables.t
+    ; locks   : String_with_vars.t list
+    ; package : Package.t option
+    ; deps    : Dep_conf.t list
+    }
+
+  let gen_parse names =
+    record
+      (Buildable.t                                         >>= fun buildable ->
+       field_oslu "link_flags"                             >>= fun link_flags ->
+       names                                               >>= fun names ->
+       field "deps" (list Dep_conf.t) ~default:[]          >>= fun deps ->
+       field_o "package" Pkg.t                             >>= fun package ->
+       field "locks" (list String_with_vars.t) ~default:[] >>= fun locks ->
+       field "modes" Executables.Link_mode.Set.t
+         ~default:Executables.Link_mode.Set.default >>= fun modes ->
+       return
+         { exes =
+             { Executables.
+               link_flags
+             ; link_deps = []
+             ; modes
+             ; buildable
+             ; names
+             }
+         ; locks
+         ; package
+         ; deps
+         })
+
+  let multi = gen_parse (field "names" (list (located string)))
+
+  let single = gen_parse (field "name" (located string) >>| List.singleton)
+end
+
 module Copy_files = struct
   type t = { add_line_directive : bool
            ; glob : String_with_vars.t
@@ -1426,6 +1463,7 @@ type Stanza.t +=
   | Copy_files  of Copy_files.t
   | Documentation of Documentation.t
   | Env         of Env.t
+  | Tests       of Tests.t
 
 module Stanzas = struct
   type t = Stanza.t list
@@ -1483,6 +1521,12 @@ module Stanzas = struct
     ; "jbuild_version",
       (Syntax.deleted_in Stanza.syntax (1, 0) >>= fun () ->
        Jbuild_version.t >>| fun _ -> [])
+    ; "tests",
+      (Syntax.since Stanza.syntax (1, 0) >>= fun () ->
+       (Tests.multi >>| fun t -> [Tests t]))
+    ; "test",
+      (Syntax.since Stanza.syntax (1, 0) >>= fun () ->
+       (Tests.single >>| fun t -> [Tests t]))
     ; "env",
       (Syntax.since Stanza.syntax (1, 0) >>= fun () ->
        loc >>= fun loc ->

src/jbuild.mli

@@ -345,6 +345,15 @@ module Env : sig
     }
 end
 
+module Tests : sig
+  type t =
+    { exes    : Executables.t
+    ; locks   : String_with_vars.t list
+    ; package : Package.t option
+    ; deps    : Dep_conf.t list
+    }
+end
+
 type Stanza.t +=
   | Library     of Library.t
   | Executables of Executables.t
@@ -354,6 +363,7 @@ type Stanza.t +=
   | Copy_files  of Copy_files.t
   | Documentation of Documentation.t
   | Env         of Env.t
+  | Tests       of Tests.t
 
 module Stanzas : sig
   type t = Stanza.t list

src/stdune/list.ml

@@ -102,3 +102,5 @@ let rec assoc t x =
   match t with
   | [] -> None
   | (k, v) :: t -> if x = k then Some v else assoc t x
+
+let singleton x = [x]

src/stdune/list.mli

@@ -38,3 +38,5 @@ val stable_sort : 'a t -> compare:('a -> 'a -> Ordering.t) -> 'a t
 val compare : 'a t -> 'a t -> compare:('a -> 'a -> Ordering.t) -> Ordering.t
 
 val assoc : ('a * 'b) t -> 'a -> 'b option
+
+val singleton : 'a -> 'a t

src/string_with_vars.ml

@@ -7,6 +7,15 @@ type t =
   ; syntax_version : Syntax.Version.t
   }
 
+let make_text ?(quoted=false) loc s =
+  { template =
+      { parts = [Text s]
+      ; quoted
+      ; loc
+      }
+  ; syntax_version = (1, 0)
+  }
+
 let literal ~quoted ~loc s =
   { parts = [Text s]
   ; quoted

src/string_with_vars.mli

@@ -27,6 +27,7 @@ val sexp_of_t : t -> Sexp.t
 val virt       : ?quoted: bool -> (string * int * int * int) -> string -> t
 val virt_var   : ?quoted: bool -> (string * int * int * int) -> string -> t
 val virt_text  : (string * int * int * int) -> string -> t
+val make_text  : ?quoted: bool -> Loc.t -> string -> t
 
 val is_var : t -> name:string -> bool
 

test/blackbox-tests/dune.inc

@@ -571,6 +571,14 @@
    test-cases/syntax-versioning
    (progn (run %{exe:cram.exe} -test run.t) (diff? run.t run.t.corrected)))))
 
+(alias
+ (name tests-stanza)
+ (deps (package dune) (source_tree test-cases/tests-stanza))
+ (action
+  (chdir
+   test-cases/tests-stanza
+   (progn (run %{exe:cram.exe} -test run.t) (diff? run.t run.t.corrected)))))
+
 (alias
  (name use-meta)
  (deps (package dune) (source_tree test-cases/use-meta))
@@ -672,6 +680,7 @@
   (alias scope-ppx-bug)
   (alias select)
   (alias syntax-versioning)
+  (alias tests-stanza)
   (alias use-meta)
   (alias utop)
   (alias windows-diff)
@@ -738,6 +747,7 @@
   (alias scope-ppx-bug)
   (alias select)
   (alias syntax-versioning)
+  (alias tests-stanza)
   (alias use-meta)
   (alias windows-diff)
   (alias workspaces)))

test/blackbox-tests/test-cases/tests-stanza/dune

@@ -0,0 +1,7 @@
+(tests
+ (names expect_test regular_test)
+ (modules :standard \ singular))
+
+(test
+ (name singular)
+ (modules singular))

test/blackbox-tests/test-cases/tests-stanza/dune-project

@@ -0,0 +1 @@
+(lang dune 1.0)

test/blackbox-tests/test-cases/tests-stanza/expect_test.expected

@@ -0,0 +1 @@
+expect test

test/blackbox-tests/test-cases/tests-stanza/expect_test.ml

@@ -0,0 +1 @@
+let () = print_endline "expect test"

test/blackbox-tests/test-cases/tests-stanza/regular_test.ml

@@ -0,0 +1 @@
+let () = print_endline "regular test"

test/blackbox-tests/test-cases/tests-stanza/run.t

@@ -0,0 +1,18 @@
+  $ dune runtest --display short
+      ocamldep .singular.eobjs/singular.ml.d
+        ocamlc .singular.eobjs/singular.{cmi,cmo,cmt}
+      ocamlopt .singular.eobjs/singular.{cmx,o}
+      ocamlopt singular.exe
+      singular alias runtest
+  singular test
+      ocamldep .expect_test.eobjs/expect_test.ml.d
+      ocamldep .expect_test.eobjs/regular_test.ml.d
+        ocamlc .expect_test.eobjs/regular_test.{cmi,cmo,cmt}
+      ocamlopt .expect_test.eobjs/regular_test.{cmx,o}
+      ocamlopt regular_test.exe
+  regular_test alias runtest
+  regular test
+        ocamlc .expect_test.eobjs/expect_test.{cmi,cmo,cmt}
+      ocamlopt .expect_test.eobjs/expect_test.{cmx,o}
+      ocamlopt expect_test.exe
+   expect_test expect_test.output

test/blackbox-tests/test-cases/tests-stanza/singular.ml

@@ -0,0 +1 @@
+print_endline "singular test"