Test Stanza Proposal (#822)
Add tests and test stanza These stanzas are used to easily define tests. If a test has a corresponding .expect file, it will be immediately considered as an expect test.
This commit is contained in:
parent
19e332d8a4
commit
7707872e54
|
@ -240,6 +240,8 @@ Executables can also be linked as object or shared object files. See
|
|||
(section bin)
|
||||
(files (<name>.exe as <public-name>)))
|
||||
|
||||
.. _shared-exe-fields:
|
||||
|
||||
- ``(package <package>)`` if there is a ``(public_name ...)`` field, this
|
||||
specifies the package the executables are part of
|
||||
|
||||
|
@ -548,6 +550,8 @@ The syntax is as follows:
|
|||
|
||||
``<name>`` is an alias name such as ``runtest``.
|
||||
|
||||
.. _alias-fields:
|
||||
|
||||
``<deps-conf list>`` specifies the dependencies of the alias. See the
|
||||
`Dependency specification`_ section for more details.
|
||||
|
||||
|
@ -698,6 +702,45 @@ With this dune file, running dune as follow will replace the
|
|||
|
||||
$ dune build @runtest --auto-promote
|
||||
|
||||
.. _tests-stanza:
|
||||
|
||||
tests
|
||||
-----
|
||||
|
||||
The ``tests`` stanza allows one to easily define multiple tests. For example we
|
||||
can define two tests at once with:
|
||||
|
||||
.. code:: scheme
|
||||
|
||||
(tests
|
||||
(names mytest expect_test)
|
||||
<optional fields>)
|
||||
|
||||
This will define an executable named ``mytest.exe`` that will be executed as
|
||||
part of the ``runtest`` alias. If the directory also contains an
|
||||
``expect_test.expected`` file, then ``expect_test`` will be used to define an
|
||||
expect test. That is, the test will be executed and its output will be compared
|
||||
to ``expect_test.expected``.
|
||||
|
||||
The optional fields that are supported are a subset of the alias and executables
|
||||
fields. In particular, all fields except for ``public_names`` are supported from
|
||||
the `executables stanza <shared-exe-fields>`_. Alias fields apart from ``name``
|
||||
and ``action`` are allowed.
|
||||
|
||||
test
|
||||
----
|
||||
|
||||
The ``test`` stanza is the singular form of ``tests``. The only difference is
|
||||
that it's of the form:
|
||||
|
||||
.. code:: scheme
|
||||
|
||||
(test
|
||||
(name foo)
|
||||
<optional fields>)
|
||||
|
||||
where the ``name`` field is singular. The same optional fields are supported.
|
||||
|
||||
.. _dune-env:
|
||||
|
||||
env
|
||||
|
|
|
@ -330,6 +330,14 @@ running your testsuite, simply add this to a jbuild file:
|
|||
((name runtest)
|
||||
(action (run ./tests.exe))))
|
||||
|
||||
Hence to define an a test a pair of alias and executable stanzas are required.
|
||||
To simplify this common pattern, dune provides a :ref:`tests-stanza` stanza to
|
||||
define multiple tests and their aliases at once:
|
||||
|
||||
.. code:: scheme
|
||||
|
||||
(tests (names test1 test2))
|
||||
|
||||
Diffing the result
|
||||
------------------
|
||||
|
||||
|
@ -348,12 +356,21 @@ command. For instance let's consider this test:
|
|||
((name runtest)
|
||||
(action (diff tests.expected test.output))))
|
||||
|
||||
After having run ``tests.exe`` and dumping its output to
|
||||
``tests.output``, dune will compare the latter to
|
||||
``tests.expected``. In case of mismatch, dune will print a diff
|
||||
and then the ``dune promote`` command can be used to copy over the
|
||||
generated ``test.output`` file to ``tests.expected`` in the source
|
||||
tree. This provides a nice way of dealing with the usual *write code*,
|
||||
After having run ``tests.exe`` and dumping its output to ``tests.output``, dune
|
||||
will compare the latter to ``tests.expected``. In case of mismatch, dune will
|
||||
print a diff and then the ``dune promote`` command can be used to copy over the
|
||||
generated ``test.output`` file to ``tests.expected`` in the source tree.
|
||||
|
||||
Alternatively, the :ref:`tests-stanza` also supports this style of tests.
|
||||
|
||||
.. code:: scheme
|
||||
|
||||
(tests (names tests))
|
||||
|
||||
Where dune expects a ``tests.expected`` file to exist to infer that this is an
|
||||
expect tests.
|
||||
|
||||
This provides a nice way of dealing with the usual *write code*,
|
||||
*run*, *promote* cycle of testing. For instance:
|
||||
|
||||
.. code:: bash
|
||||
|
|
|
@ -953,6 +953,67 @@ module Gen(P : Install_rules.Params) = struct
|
|||
~targets:Alias
|
||||
~scope)
|
||||
|
||||
let tests_rules (t : Tests.t) ~dir ~scope ~all_modules ~modules_partitioner
|
||||
~dir_kind ~src_dir =
|
||||
let test_kind (loc, name) =
|
||||
let sources = SC.source_files sctx ~src_path:src_dir in
|
||||
let expected_basename = name ^ ".expected" in
|
||||
if String.Set.mem sources expected_basename then
|
||||
`Expect
|
||||
{ Action.Unexpanded.Diff.
|
||||
file1 = String_with_vars.make_text loc expected_basename
|
||||
; file2 = String_with_vars.make_text loc (name ^ ".output")
|
||||
; optional = false
|
||||
; mode = Text
|
||||
}
|
||||
else
|
||||
`Regular
|
||||
in
|
||||
let regular_rule run_action alias loc =
|
||||
{ alias with Alias_conf.action = Some (loc, run_action) }
|
||||
in
|
||||
let expect_rule run_action (diff : Action.Unexpanded.Diff.t) alias loc =
|
||||
let rule =
|
||||
{ Rule.
|
||||
targets = Infer
|
||||
; deps = []
|
||||
; action =
|
||||
(loc, Action.Unexpanded.Redirect (Stdout, diff.file2, run_action))
|
||||
; mode = Standard
|
||||
; locks = t.locks
|
||||
; loc
|
||||
} in
|
||||
let alias =
|
||||
{ alias with
|
||||
Alias_conf.
|
||||
action = Some (loc, Diff diff)
|
||||
; locks = t.locks
|
||||
} in
|
||||
(alias, rule)
|
||||
in
|
||||
List.iter t.exes.names ~f:(fun (loc, s) ->
|
||||
let run_action =
|
||||
Action.Unexpanded.Run
|
||||
(String_with_vars.make_text loc ("./" ^ s ^ ".exe"), []) in
|
||||
let base_alias =
|
||||
{ Alias_conf.
|
||||
name = "runtest"
|
||||
; locks = []
|
||||
; package = t.package
|
||||
; deps = t.deps
|
||||
; action = None
|
||||
} in
|
||||
match test_kind (loc, s) with
|
||||
| `Regular ->
|
||||
alias_rules ~dir ~scope (regular_rule run_action base_alias loc)
|
||||
| `Expect diff ->
|
||||
let (alias, rule) =
|
||||
expect_rule run_action diff base_alias loc in
|
||||
alias_rules ~dir ~scope alias;
|
||||
ignore (user_rule ~dir ~scope rule : Path.t list));
|
||||
executables_rules t.exes ~dir ~all_modules ~scope ~dir_kind
|
||||
~modules_partitioner
|
||||
|
||||
(* +-----------------------------------------------------------------+
|
||||
| Stanza |
|
||||
+-----------------------------------------------------------------+ *)
|
||||
|
@ -975,6 +1036,9 @@ module Gen(P : Install_rules.Params) = struct
|
|||
| Alias alias ->
|
||||
alias_rules alias ~dir ~scope;
|
||||
None
|
||||
| Tests tests ->
|
||||
Some (tests_rules tests ~dir ~scope ~all_modules ~src_dir
|
||||
~modules_partitioner ~dir_kind:kind)
|
||||
| Copy_files { glob; _ } ->
|
||||
let src_dir =
|
||||
let loc = String_with_vars.loc glob in
|
||||
|
@ -1011,7 +1075,7 @@ module Gen(P : Install_rules.Params) = struct
|
|||
| _ -> ());
|
||||
Modules_partitioner.emit_errors modules_partitioner
|
||||
|
||||
let gen_rules ~dir components : Build_system.extra_sub_directories_to_keep =
|
||||
let gen_rules ~dir components : Build_system.extra_sub_directories_to_keep =
|
||||
(match components with
|
||||
| ".js" :: rest -> Js_of_ocaml_rules.setup_separate_compilation_rules
|
||||
sctx rest
|
||||
|
|
|
@ -1352,6 +1352,43 @@ module Alias_conf = struct
|
|||
})
|
||||
end
|
||||
|
||||
module Tests = struct
|
||||
type t =
|
||||
{ exes : Executables.t
|
||||
; locks : String_with_vars.t list
|
||||
; package : Package.t option
|
||||
; deps : Dep_conf.t list
|
||||
}
|
||||
|
||||
let gen_parse names =
|
||||
record
|
||||
(Buildable.t >>= fun buildable ->
|
||||
field_oslu "link_flags" >>= fun link_flags ->
|
||||
names >>= fun names ->
|
||||
field "deps" (list Dep_conf.t) ~default:[] >>= fun deps ->
|
||||
field_o "package" Pkg.t >>= fun package ->
|
||||
field "locks" (list String_with_vars.t) ~default:[] >>= fun locks ->
|
||||
field "modes" Executables.Link_mode.Set.t
|
||||
~default:Executables.Link_mode.Set.default >>= fun modes ->
|
||||
return
|
||||
{ exes =
|
||||
{ Executables.
|
||||
link_flags
|
||||
; link_deps = []
|
||||
; modes
|
||||
; buildable
|
||||
; names
|
||||
}
|
||||
; locks
|
||||
; package
|
||||
; deps
|
||||
})
|
||||
|
||||
let multi = gen_parse (field "names" (list (located string)))
|
||||
|
||||
let single = gen_parse (field "name" (located string) >>| List.singleton)
|
||||
end
|
||||
|
||||
module Copy_files = struct
|
||||
type t = { add_line_directive : bool
|
||||
; glob : String_with_vars.t
|
||||
|
@ -1426,6 +1463,7 @@ type Stanza.t +=
|
|||
| Copy_files of Copy_files.t
|
||||
| Documentation of Documentation.t
|
||||
| Env of Env.t
|
||||
| Tests of Tests.t
|
||||
|
||||
module Stanzas = struct
|
||||
type t = Stanza.t list
|
||||
|
@ -1483,6 +1521,12 @@ module Stanzas = struct
|
|||
; "jbuild_version",
|
||||
(Syntax.deleted_in Stanza.syntax (1, 0) >>= fun () ->
|
||||
Jbuild_version.t >>| fun _ -> [])
|
||||
; "tests",
|
||||
(Syntax.since Stanza.syntax (1, 0) >>= fun () ->
|
||||
(Tests.multi >>| fun t -> [Tests t]))
|
||||
; "test",
|
||||
(Syntax.since Stanza.syntax (1, 0) >>= fun () ->
|
||||
(Tests.single >>| fun t -> [Tests t]))
|
||||
; "env",
|
||||
(Syntax.since Stanza.syntax (1, 0) >>= fun () ->
|
||||
loc >>= fun loc ->
|
||||
|
|
|
@ -345,6 +345,15 @@ module Env : sig
|
|||
}
|
||||
end
|
||||
|
||||
module Tests : sig
|
||||
type t =
|
||||
{ exes : Executables.t
|
||||
; locks : String_with_vars.t list
|
||||
; package : Package.t option
|
||||
; deps : Dep_conf.t list
|
||||
}
|
||||
end
|
||||
|
||||
type Stanza.t +=
|
||||
| Library of Library.t
|
||||
| Executables of Executables.t
|
||||
|
@ -354,6 +363,7 @@ type Stanza.t +=
|
|||
| Copy_files of Copy_files.t
|
||||
| Documentation of Documentation.t
|
||||
| Env of Env.t
|
||||
| Tests of Tests.t
|
||||
|
||||
module Stanzas : sig
|
||||
type t = Stanza.t list
|
||||
|
|
|
@ -102,3 +102,5 @@ let rec assoc t x =
|
|||
match t with
|
||||
| [] -> None
|
||||
| (k, v) :: t -> if x = k then Some v else assoc t x
|
||||
|
||||
let singleton x = [x]
|
||||
|
|
|
@ -38,3 +38,5 @@ val stable_sort : 'a t -> compare:('a -> 'a -> Ordering.t) -> 'a t
|
|||
val compare : 'a t -> 'a t -> compare:('a -> 'a -> Ordering.t) -> Ordering.t
|
||||
|
||||
val assoc : ('a * 'b) t -> 'a -> 'b option
|
||||
|
||||
val singleton : 'a -> 'a t
|
||||
|
|
|
@ -7,6 +7,15 @@ type t =
|
|||
; syntax_version : Syntax.Version.t
|
||||
}
|
||||
|
||||
let make_text ?(quoted=false) loc s =
|
||||
{ template =
|
||||
{ parts = [Text s]
|
||||
; quoted
|
||||
; loc
|
||||
}
|
||||
; syntax_version = (1, 0)
|
||||
}
|
||||
|
||||
let literal ~quoted ~loc s =
|
||||
{ parts = [Text s]
|
||||
; quoted
|
||||
|
|
|
@ -27,6 +27,7 @@ val sexp_of_t : t -> Sexp.t
|
|||
val virt : ?quoted: bool -> (string * int * int * int) -> string -> t
|
||||
val virt_var : ?quoted: bool -> (string * int * int * int) -> string -> t
|
||||
val virt_text : (string * int * int * int) -> string -> t
|
||||
val make_text : ?quoted: bool -> Loc.t -> string -> t
|
||||
|
||||
val is_var : t -> name:string -> bool
|
||||
|
||||
|
|
|
@ -571,6 +571,14 @@
|
|||
test-cases/syntax-versioning
|
||||
(progn (run %{exe:cram.exe} -test run.t) (diff? run.t run.t.corrected)))))
|
||||
|
||||
(alias
|
||||
(name tests-stanza)
|
||||
(deps (package dune) (source_tree test-cases/tests-stanza))
|
||||
(action
|
||||
(chdir
|
||||
test-cases/tests-stanza
|
||||
(progn (run %{exe:cram.exe} -test run.t) (diff? run.t run.t.corrected)))))
|
||||
|
||||
(alias
|
||||
(name use-meta)
|
||||
(deps (package dune) (source_tree test-cases/use-meta))
|
||||
|
@ -672,6 +680,7 @@
|
|||
(alias scope-ppx-bug)
|
||||
(alias select)
|
||||
(alias syntax-versioning)
|
||||
(alias tests-stanza)
|
||||
(alias use-meta)
|
||||
(alias utop)
|
||||
(alias windows-diff)
|
||||
|
@ -738,6 +747,7 @@
|
|||
(alias scope-ppx-bug)
|
||||
(alias select)
|
||||
(alias syntax-versioning)
|
||||
(alias tests-stanza)
|
||||
(alias use-meta)
|
||||
(alias windows-diff)
|
||||
(alias workspaces)))
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
(tests
|
||||
(names expect_test regular_test)
|
||||
(modules :standard \ singular))
|
||||
|
||||
(test
|
||||
(name singular)
|
||||
(modules singular))
|
|
@ -0,0 +1 @@
|
|||
(lang dune 1.0)
|
|
@ -0,0 +1 @@
|
|||
expect test
|
|
@ -0,0 +1 @@
|
|||
let () = print_endline "expect test"
|
|
@ -0,0 +1 @@
|
|||
let () = print_endline "regular test"
|
|
@ -0,0 +1,18 @@
|
|||
$ dune runtest --display short
|
||||
ocamldep .singular.eobjs/singular.ml.d
|
||||
ocamlc .singular.eobjs/singular.{cmi,cmo,cmt}
|
||||
ocamlopt .singular.eobjs/singular.{cmx,o}
|
||||
ocamlopt singular.exe
|
||||
singular alias runtest
|
||||
singular test
|
||||
ocamldep .expect_test.eobjs/expect_test.ml.d
|
||||
ocamldep .expect_test.eobjs/regular_test.ml.d
|
||||
ocamlc .expect_test.eobjs/regular_test.{cmi,cmo,cmt}
|
||||
ocamlopt .expect_test.eobjs/regular_test.{cmx,o}
|
||||
ocamlopt regular_test.exe
|
||||
regular_test alias runtest
|
||||
regular test
|
||||
ocamlc .expect_test.eobjs/expect_test.{cmi,cmo,cmt}
|
||||
ocamlopt .expect_test.eobjs/expect_test.{cmx,o}
|
||||
ocamlopt expect_test.exe
|
||||
expect_test expect_test.output
|
|
@ -0,0 +1 @@
|
|||
print_endline "singular test"
|
Loading…
Reference in New Issue