From be3840005e63ebb5cd3980a280ee09ac428df57a Mon Sep 17 00:00:00 2001
From: Rudi Grinberg <rudi.grinberg@gmail.com>
Date: Fri, 20 Apr 2018 17:11:32 +0700
Subject: [PATCH] Generate blackbox tests

Generate these tests from the listing of directories. This allows us to
effectively mark which tests are not working in certain conditions/platforms.
---
 test/blackbox-tests/cram.mll                  |  30 +-
 test/blackbox-tests/gen_tests.ml              | 137 +++++
 test/blackbox-tests/jbuild                    | 509 +----------------
 test/blackbox-tests/jbuild.inc                | 526 ++++++++++++++++++
 test/blackbox-tests/platform.ml               |  30 +
 .../test-cases/github25/root/run.t            |  24 -
 test/blackbox-tests/test-cases/github25/run.t |  27 +
 7 files changed, 758 insertions(+), 525 deletions(-)
 create mode 100644 test/blackbox-tests/gen_tests.ml
 create mode 100644 test/blackbox-tests/jbuild.inc
 create mode 100644 test/blackbox-tests/platform.ml
 delete mode 100644 test/blackbox-tests/test-cases/github25/root/run.t
 create mode 100644 test/blackbox-tests/test-cases/github25/run.t

diff --git a/test/blackbox-tests/cram.mll b/test/blackbox-tests/cram.mll
index 95510d2b..5fe4cd0a 100644
--- a/test/blackbox-tests/cram.mll
+++ b/test/blackbox-tests/cram.mll
@@ -100,27 +100,35 @@ and postprocess tbl b = parse
 
   let () =
     let skip_versions = ref [] in
+    let skip_platforms = ref [] in
     let expect_test = ref None in
     let args =
-      [ "-skip-versions"
-      , Arg.String (fun s -> skip_versions := parse_skip_versions s)
-      , "Comma separated versions of ocaml where to skip test"
-      ; "-test"
-      , Arg.String (fun s -> expect_test := Some s)
-      , "expect test file"
-      ] in
+      (Platform.argv skip_platforms)
+      :: [ "-skip-versions"
+         , Arg.String (fun s -> skip_versions := parse_skip_versions s)
+         , "Comma separated versions of ocaml where to skip test"
+         ; "-test"
+         , Arg.String (fun s -> expect_test := Some s)
+         , "expect test file"
+         ] in
     Configurator.main ~args ~name:"cram" (fun configurator ->
       let expect_test =
         match !expect_test with
         | None -> raise (Arg.Bad "expect test file must be passed")
         | Some p -> p in
-      let ocaml_version =
-        Configurator.ocaml_config_var_exn configurator "version"
-        |> parse_version in
       begin
+        let ocaml_version =
+          Configurator.ocaml_config_var_exn configurator "version"
+          |> parse_version in
         if List.exists !skip_versions ~f:(fun (op, v') ->
           test op ocaml_version v') then
-          exit 0
+          exit 0;
+        let platform =
+          Configurator.ocaml_config_var_exn configurator "system" in
+        if List.exists !skip_platforms ~f:(fun p ->
+          Platform.to_string p = platform
+        ) then
+          exit 0;
       end;
       Test_common.run_expect_test expect_test ~f:(fun file_contents lexbuf ->
         let items = file lexbuf in
diff --git a/test/blackbox-tests/gen_tests.ml b/test/blackbox-tests/gen_tests.ml
new file mode 100644
index 00000000..2abdbc9f
--- /dev/null
+++ b/test/blackbox-tests/gen_tests.ml
@@ -0,0 +1,137 @@
+open Stdune
+
+let sprintf = Printf.sprintf
+
+module Sexp = struct
+  let record fields =
+    Usexp.List
+      (List.map fields ~f:(fun (k, s) ->
+         Usexp.List [Usexp.atom k; s]))
+
+  let strings fields =
+    Usexp.List (List.map fields ~f:Usexp.atom_or_quoted_string)
+
+  let constr name args =
+    Usexp.List [Usexp.atom name; args]
+end
+
+let alias ?action name ~deps =
+  Sexp.constr "alias"
+    (Sexp.record (
+       [ "name", Usexp.atom name
+       ; "deps", Usexp.List deps
+       ] @ (match action with
+         | None -> []
+         | Some a -> ["action", a])))
+
+module Test = struct
+  type t =
+    { name           : string
+    ; env            : (string * string) option
+    ; skip_ocaml     : string option
+    ; skip_platforms : Platform.t list
+    ; enabled        : bool
+    ; js             : bool
+    ; external_deps  : bool
+    }
+
+  let make ?env ?skip_ocaml ?(skip_platforms=[]) ?(enabled=true) ?(js=false)
+        ?(external_deps=false) name =
+    { name
+    ; env
+    ; skip_ocaml
+    ; skip_platforms
+    ; external_deps
+    ; enabled
+    ; js
+    }
+
+  let pp_sexp fmt t =
+    let open Usexp in
+    let skip_version =
+      match t.skip_ocaml with
+      | None -> []
+      | Some s -> ["-skip-versions"; s]
+    in
+    let skip_platforms = Platform.to_cmd t.skip_platforms in
+    let action =
+      List
+        [ atom "chdir"
+        ; atom (sprintf "test-cases/%s" t.name)
+        ; List
+            [ atom "progn"
+            ; Sexp.strings (["run"; "${exe:cram.exe}"]
+                            @ skip_version
+                            @ skip_platforms
+                            @ ["-test"; "run.t"])
+            ; Sexp.strings ["diff?"; "run.t"; "run.t.corrected"]
+            ]
+
+        ]
+    in
+    let action =
+      match t.env with
+      | None -> action
+      | Some (k, v) ->
+        List [ atom "setenv"
+             ; atom_or_quoted_string k
+             ; atom_or_quoted_string v
+             ; action ] in
+    alias t.name
+      ~deps:(
+        [ Sexp.strings ["package"; "jbuilder"]
+        ; Sexp.strings [ "files_recursively_in"
+                       ; sprintf "test-cases/%s" t.name]
+        ]
+      ) ~action
+    |> Usexp.pp fmt
+end
+
+let exclusions =
+  let open Test in
+  [ make "js_of_ocaml" ~js:true ~env:("NODE", "${bin:node}")
+  ; make "github25" ~env:("OCAMLPATH", "../findlib-packages")
+  ; make "odoc" ~external_deps:true ~skip_ocaml:"4.02.3"
+  ; make "multiple-private-libs"  ~skip_ocaml:"4.02.3"
+  ; make "ppx-rewriter" ~skip_ocaml:"4.02.3" ~external_deps:true
+  ; make "odoc-unique-mlds" ~skip_ocaml:"4.02.3" ~external_deps:true
+  ; make "output-obj" ~skip_platforms:[Mac; Win]
+  ; make "github644" ~external_deps:true
+  ; make "private-public-overlap" ~external_deps:true
+  ; make "reason" ~enabled:false
+  ; make "menhir"~external_deps:true
+  ; make "configurator" ~skip_platforms:[Win]
+  ]
+
+let all_tests = lazy (
+  Sys.readdir "test-cases"
+  |> Array.to_list
+  |> List.filter ~f:(fun s -> not (String.contains s '.'))
+  |> List.sort ~compare:String.compare
+  |> List.map ~f:(fun name ->
+    match List.find exclusions ~f:(fun (t : Test.t) -> t.name = name) with
+    | None -> Test.make name
+    | Some t -> t
+  )
+)
+
+let pp_group fmt (name, tests) =
+  alias name ~deps:(
+    (List.map tests ~f:(fun (t : Test.t) ->
+       Sexp.strings ["alias"; t.name])))
+  |> Usexp.pp fmt
+
+let () =
+  let tests = Lazy.force all_tests in
+  (* The runtest target has a "specoial" definition. It includes all tests
+     except for js and disabled tests *)
+  tests |> List.iter ~f:(fun t -> Format.printf "%a@.@." Test.pp_sexp t);
+  [ "runtest", (fun (t : Test.t) -> not t.js && t.enabled)
+  ; "runtest-no-deps", (fun (t : Test.t) -> not t.external_deps && t.enabled)
+  ; "runtest-disabled", (fun (t : Test.t) -> not t.enabled)
+  ; "runtest-js", (fun (t : Test.t) -> t.js && t.enabled) ]
+  |> List.map ~f:(fun (name, predicate) ->
+    (name, List.filter tests ~f:predicate))
+  |> Format.pp_print_list
+       ~pp_sep:(fun fmt () -> Format.fprintf fmt "@.@.")
+       pp_group Format.std_formatter
diff --git a/test/blackbox-tests/jbuild b/test/blackbox-tests/jbuild
index 319530b8..da7ec349 100644
--- a/test/blackbox-tests/jbuild
+++ b/test/blackbox-tests/jbuild
@@ -1,500 +1,29 @@
 (jbuild_version 1)
 
+(library
+ ((name platform)
+  (modules (platform))
+  (libraries (stdune))))
+
 (executable
  ((name cram)
-  (libraries (test_common jbuilder configurator))))
+  (modules (cram))
+  (libraries (test_common platform jbuilder configurator))))
 
 (ocamllex (cram))
 
-(alias
- ((name   runtest)
-  (deps   ((package jbuilder)
-           (files_recursively_in test-cases/redirections)))
-  (action
-   (chdir test-cases/redirections
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
+(executable
+ ((name gen_tests)
+  (libraries (stdune platform usexp))
+  (modules (gen_tests))))
+
+(include jbuild.inc)
+
+(rule
+ ((targets (jbuild.inc.gen))
+  (deps ((files_recursively_in test-cases)))
+  (action (with-stdout-to ${@} (run ./gen_tests.exe)))))
 
 (alias
  ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/misc)))
-  (action
-   (chdir test-cases/misc
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/github20)))
-  (action
-   (chdir test-cases/github20
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest-js)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/js_of_ocaml)))
-  (action
-   (chdir test-cases/js_of_ocaml
-    (setenv NODE ${bin:node}
-     (progn
-      (run ${exe:cram.exe} -test run.t)
-      (diff? run.t run.t.corrected)))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/github24)))
-  (action
-   (chdir test-cases/github24
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/menhir)))
-  (action
-   (chdir test-cases/menhir
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/github25)))
-  (action
-   (chdir test-cases/github25/root
-    (setenv OCAMLPATH ../findlib-packages
-     (progn
-      (run ${exe:cram.exe} -test run.t)
-      (diff? run.t run.t.corrected)))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/lib-available)))
-  (action
-   (chdir test-cases/lib-available
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/copy_files)))
-  (action
-   (chdir test-cases/copy_files
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/aliases)))
-  (action
-   (chdir test-cases/aliases
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/force-test)))
-  (action
-   (chdir test-cases/force-test
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/meta-gen)))
-  (action
-   (chdir test-cases/meta-gen
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/exec-cmd)))
-  (action
-   (chdir test-cases/exec-cmd
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/ocaml-syntax)))
-  (action
-   (chdir test-cases/ocaml-syntax
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/gen-opam-install-file)))
-  (action
-   (chdir test-cases/gen-opam-install-file
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest-skipped)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/reason)))
-  (action
-   (chdir test-cases/reason
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/odoc)))
-  (action
-   (chdir test-cases/odoc
-    (progn
-     (run ${exe:cram.exe} -skip-versions 4.02.3 -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/select)))
-  (action
-   (chdir test-cases/select
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/multiple-private-libs)))
-  (action
-   (chdir test-cases/multiple-private-libs
-    (progn
-     (run ${exe:cram.exe} -skip-versions 4.02.3 -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/ppx-rewriter)))
-  (action
-   (chdir test-cases/ppx-rewriter
-    (progn
-     (run ${exe:cram.exe} -skip-versions 4.02.3 -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/utop)))
-  (action
-   (chdir test-cases/utop
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/c-stubs)))
-  (action
-   (chdir test-cases/c-stubs
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/cross-compilation)))
-  (action
-   (chdir test-cases/cross-compilation
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/promote)))
-  (action
-   (chdir test-cases/promote
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/include-loop)))
-  (action
-   (chdir test-cases/include-loop
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/scope-bug)))
-  (action
-   (chdir test-cases/scope-bug
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/scope-ppx-bug)))
-  (action
-   (chdir test-cases/scope-ppx-bug
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/loop)))
-  (action
-   (chdir test-cases/loop
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/intf-only)))
-  (action
-   (chdir test-cases/intf-only
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/installable-dup-private-libs)))
-  (action
-   (chdir test-cases/installable-dup-private-libs
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/inline_tests)))
-  (action
-   (chdir test-cases/inline_tests
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/byte-code-only)))
-  (action
-   (chdir test-cases/byte-code-only
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/merlin-tests)))
-  (action
-   (chdir test-cases/merlin-tests
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/findlib)))
-  (action
-   (chdir test-cases/findlib
-    (setenv OCAMLPATH findlib-packages
-     (progn
-      (run ${exe:cram.exe} -test run.t)
-      (diff? run.t run.t.corrected)))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/odoc-unique-mlds)))
-  (action
-   (chdir test-cases/odoc-unique-mlds
-    (progn
-     (run ${exe:cram.exe} -skip-versions 4.02.3 -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/ocamldep-multi-stanzas)))
-  (action
-   (chdir test-cases/ocamldep-multi-stanzas
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/exclude-missing-module)))
-  (action
-   (chdir test-cases/exclude-missing-module
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/github534)))
-  (action
-   (chdir test-cases/github534
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/github568)))
-  (action
-   (chdir test-cases/github568
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/output-obj)))
-  (action
-   (chdir test-cases/output-obj
-    (progn
-     (run ${exe:cram.exe} -skip-versions <4.06.0 -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/github597)))
-  (action
-   (chdir test-cases/github597
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/private-public-overlap)))
-  (action
-   (chdir test-cases/private-public-overlap
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/depend-on-the-universe)))
-  (action
-   (chdir test-cases/depend-on-the-universe
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/package-dep)))
-  (action
-   (chdir test-cases/package-dep
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/github644)))
-  (action
-   (chdir test-cases/github644
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/configurator)))
-  (action
-   (chdir test-cases/configurator
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/quoting)))
-  (action
-   (chdir test-cases/quoting
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/null-dep)))
-  (action
-   (chdir test-cases/null-dep
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
-
-(alias
- ((name runtest)
-  (deps ((package jbuilder)
-         (files_recursively_in test-cases/github660)))
-  (action
-   (chdir test-cases/github660
-    (progn
-     (run ${exe:cram.exe} -test run.t)
-     (diff? run.t run.t.corrected))))))
+  (action (diff jbuild.inc jbuild.inc.gen))))
diff --git a/test/blackbox-tests/jbuild.inc b/test/blackbox-tests/jbuild.inc
new file mode 100644
index 00000000..8de03d72
--- /dev/null
+++ b/test/blackbox-tests/jbuild.inc
@@ -0,0 +1,526 @@
+(alias
+ ((name aliases)
+  (deps ((package jbuilder) (files_recursively_in test-cases/aliases)))
+  (action
+   (chdir
+    test-cases/aliases
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name byte-code-only)
+  (deps ((package jbuilder) (files_recursively_in test-cases/byte-code-only)))
+  (action
+   (chdir
+    test-cases/byte-code-only
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name c-stubs)
+  (deps ((package jbuilder) (files_recursively_in test-cases/c-stubs)))
+  (action
+   (chdir
+    test-cases/c-stubs
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name configurator)
+  (deps ((package jbuilder) (files_recursively_in test-cases/configurator)))
+  (action
+   (chdir
+    test-cases/configurator
+    (progn
+     (run ${exe:cram.exe} -skip-platforms win -test run.t)
+     (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name copy_files)
+  (deps ((package jbuilder) (files_recursively_in test-cases/copy_files)))
+  (action
+   (chdir
+    test-cases/copy_files
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name cross-compilation)
+  (deps
+   ((package jbuilder) (files_recursively_in test-cases/cross-compilation)))
+  (action
+   (chdir
+    test-cases/cross-compilation
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name depend-on-the-universe)
+  (deps
+   ((package jbuilder)
+    (files_recursively_in test-cases/depend-on-the-universe)))
+  (action
+   (chdir
+    test-cases/depend-on-the-universe
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name exclude-missing-module)
+  (deps
+   ((package jbuilder)
+    (files_recursively_in test-cases/exclude-missing-module)))
+  (action
+   (chdir
+    test-cases/exclude-missing-module
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name exec-cmd)
+  (deps ((package jbuilder) (files_recursively_in test-cases/exec-cmd)))
+  (action
+   (chdir
+    test-cases/exec-cmd
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name findlib)
+  (deps ((package jbuilder) (files_recursively_in test-cases/findlib)))
+  (action
+   (chdir
+    test-cases/findlib
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name force-test)
+  (deps ((package jbuilder) (files_recursively_in test-cases/force-test)))
+  (action
+   (chdir
+    test-cases/force-test
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name gen-opam-install-file)
+  (deps
+   ((package jbuilder)
+    (files_recursively_in test-cases/gen-opam-install-file)))
+  (action
+   (chdir
+    test-cases/gen-opam-install-file
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name github20)
+  (deps ((package jbuilder) (files_recursively_in test-cases/github20)))
+  (action
+   (chdir
+    test-cases/github20
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name github24)
+  (deps ((package jbuilder) (files_recursively_in test-cases/github24)))
+  (action
+   (chdir
+    test-cases/github24
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name github25)
+  (deps ((package jbuilder) (files_recursively_in test-cases/github25)))
+  (action
+   (setenv
+    OCAMLPATH
+    ../findlib-packages
+    (chdir
+     test-cases/github25
+     (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected)))))))
+
+(alias
+ ((name github534)
+  (deps ((package jbuilder) (files_recursively_in test-cases/github534)))
+  (action
+   (chdir
+    test-cases/github534
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name github568)
+  (deps ((package jbuilder) (files_recursively_in test-cases/github568)))
+  (action
+   (chdir
+    test-cases/github568
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name github597)
+  (deps ((package jbuilder) (files_recursively_in test-cases/github597)))
+  (action
+   (chdir
+    test-cases/github597
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name github644)
+  (deps ((package jbuilder) (files_recursively_in test-cases/github644)))
+  (action
+   (chdir
+    test-cases/github644
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name github660)
+  (deps ((package jbuilder) (files_recursively_in test-cases/github660)))
+  (action
+   (chdir
+    test-cases/github660
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name include-loop)
+  (deps ((package jbuilder) (files_recursively_in test-cases/include-loop)))
+  (action
+   (chdir
+    test-cases/include-loop
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name inline_tests)
+  (deps ((package jbuilder) (files_recursively_in test-cases/inline_tests)))
+  (action
+   (chdir
+    test-cases/inline_tests
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name installable-dup-private-libs)
+  (deps
+   ((package jbuilder)
+    (files_recursively_in test-cases/installable-dup-private-libs)))
+  (action
+   (chdir
+    test-cases/installable-dup-private-libs
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name intf-only)
+  (deps ((package jbuilder) (files_recursively_in test-cases/intf-only)))
+  (action
+   (chdir
+    test-cases/intf-only
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name js_of_ocaml)
+  (deps ((package jbuilder) (files_recursively_in test-cases/js_of_ocaml)))
+  (action
+   (setenv
+    NODE
+    ${bin:node}
+    (chdir
+     test-cases/js_of_ocaml
+     (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected)))))))
+
+(alias
+ ((name lib-available)
+  (deps ((package jbuilder) (files_recursively_in test-cases/lib-available)))
+  (action
+   (chdir
+    test-cases/lib-available
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name loop)
+  (deps ((package jbuilder) (files_recursively_in test-cases/loop)))
+  (action
+   (chdir
+    test-cases/loop
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name menhir)
+  (deps ((package jbuilder) (files_recursively_in test-cases/menhir)))
+  (action
+   (chdir
+    test-cases/menhir
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name merlin-tests)
+  (deps ((package jbuilder) (files_recursively_in test-cases/merlin-tests)))
+  (action
+   (chdir
+    test-cases/merlin-tests
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name meta-gen)
+  (deps ((package jbuilder) (files_recursively_in test-cases/meta-gen)))
+  (action
+   (chdir
+    test-cases/meta-gen
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name misc)
+  (deps ((package jbuilder) (files_recursively_in test-cases/misc)))
+  (action
+   (chdir
+    test-cases/misc
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name multiple-private-libs)
+  (deps
+   ((package jbuilder)
+    (files_recursively_in test-cases/multiple-private-libs)))
+  (action
+   (chdir
+    test-cases/multiple-private-libs
+    (progn
+     (run ${exe:cram.exe} -skip-versions 4.02.3 -test run.t)
+     (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name null-dep)
+  (deps ((package jbuilder) (files_recursively_in test-cases/null-dep)))
+  (action
+   (chdir
+    test-cases/null-dep
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name ocaml-syntax)
+  (deps ((package jbuilder) (files_recursively_in test-cases/ocaml-syntax)))
+  (action
+   (chdir
+    test-cases/ocaml-syntax
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name ocamldep-multi-stanzas)
+  (deps
+   ((package jbuilder)
+    (files_recursively_in test-cases/ocamldep-multi-stanzas)))
+  (action
+   (chdir
+    test-cases/ocamldep-multi-stanzas
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name odoc)
+  (deps ((package jbuilder) (files_recursively_in test-cases/odoc)))
+  (action
+   (chdir
+    test-cases/odoc
+    (progn
+     (run ${exe:cram.exe} -skip-versions 4.02.3 -test run.t)
+     (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name odoc-unique-mlds)
+  (deps
+   ((package jbuilder) (files_recursively_in test-cases/odoc-unique-mlds)))
+  (action
+   (chdir
+    test-cases/odoc-unique-mlds
+    (progn
+     (run ${exe:cram.exe} -skip-versions 4.02.3 -test run.t)
+     (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name output-obj)
+  (deps ((package jbuilder) (files_recursively_in test-cases/output-obj)))
+  (action
+   (chdir
+    test-cases/output-obj
+    (progn
+     (run ${exe:cram.exe} -skip-platforms macosx,win -test run.t)
+     (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name package-dep)
+  (deps ((package jbuilder) (files_recursively_in test-cases/package-dep)))
+  (action
+   (chdir
+    test-cases/package-dep
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name ppx-rewriter)
+  (deps ((package jbuilder) (files_recursively_in test-cases/ppx-rewriter)))
+  (action
+   (chdir
+    test-cases/ppx-rewriter
+    (progn
+     (run ${exe:cram.exe} -skip-versions 4.02.3 -test run.t)
+     (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name private-public-overlap)
+  (deps
+   ((package jbuilder)
+    (files_recursively_in test-cases/private-public-overlap)))
+  (action
+   (chdir
+    test-cases/private-public-overlap
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name promote)
+  (deps ((package jbuilder) (files_recursively_in test-cases/promote)))
+  (action
+   (chdir
+    test-cases/promote
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name quoting)
+  (deps ((package jbuilder) (files_recursively_in test-cases/quoting)))
+  (action
+   (chdir
+    test-cases/quoting
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name reason)
+  (deps ((package jbuilder) (files_recursively_in test-cases/reason)))
+  (action
+   (chdir
+    test-cases/reason
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name redirections)
+  (deps ((package jbuilder) (files_recursively_in test-cases/redirections)))
+  (action
+   (chdir
+    test-cases/redirections
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name scope-bug)
+  (deps ((package jbuilder) (files_recursively_in test-cases/scope-bug)))
+  (action
+   (chdir
+    test-cases/scope-bug
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name scope-ppx-bug)
+  (deps ((package jbuilder) (files_recursively_in test-cases/scope-ppx-bug)))
+  (action
+   (chdir
+    test-cases/scope-ppx-bug
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name select)
+  (deps ((package jbuilder) (files_recursively_in test-cases/select)))
+  (action
+   (chdir
+    test-cases/select
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name utop)
+  (deps ((package jbuilder) (files_recursively_in test-cases/utop)))
+  (action
+   (chdir
+    test-cases/utop
+    (progn (run ${exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))))
+
+(alias
+ ((name runtest)
+  (deps
+   ((alias aliases)
+    (alias byte-code-only)
+    (alias c-stubs)
+    (alias configurator)
+    (alias copy_files)
+    (alias cross-compilation)
+    (alias depend-on-the-universe)
+    (alias exclude-missing-module)
+    (alias exec-cmd)
+    (alias findlib)
+    (alias force-test)
+    (alias gen-opam-install-file)
+    (alias github20)
+    (alias github24)
+    (alias github25)
+    (alias github534)
+    (alias github568)
+    (alias github597)
+    (alias github644)
+    (alias github660)
+    (alias include-loop)
+    (alias inline_tests)
+    (alias installable-dup-private-libs)
+    (alias intf-only)
+    (alias lib-available)
+    (alias loop)
+    (alias menhir)
+    (alias merlin-tests)
+    (alias meta-gen)
+    (alias misc)
+    (alias multiple-private-libs)
+    (alias null-dep)
+    (alias ocaml-syntax)
+    (alias ocamldep-multi-stanzas)
+    (alias odoc)
+    (alias odoc-unique-mlds)
+    (alias output-obj)
+    (alias package-dep)
+    (alias ppx-rewriter)
+    (alias private-public-overlap)
+    (alias promote)
+    (alias quoting)
+    (alias redirections)
+    (alias scope-bug)
+    (alias scope-ppx-bug)
+    (alias select)
+    (alias utop)))))
+
+(alias
+ ((name runtest-no-deps)
+  (deps
+   ((alias aliases)
+    (alias byte-code-only)
+    (alias c-stubs)
+    (alias configurator)
+    (alias copy_files)
+    (alias cross-compilation)
+    (alias depend-on-the-universe)
+    (alias exclude-missing-module)
+    (alias exec-cmd)
+    (alias findlib)
+    (alias force-test)
+    (alias gen-opam-install-file)
+    (alias github20)
+    (alias github24)
+    (alias github25)
+    (alias github534)
+    (alias github568)
+    (alias github597)
+    (alias github660)
+    (alias include-loop)
+    (alias inline_tests)
+    (alias installable-dup-private-libs)
+    (alias intf-only)
+    (alias js_of_ocaml)
+    (alias lib-available)
+    (alias loop)
+    (alias merlin-tests)
+    (alias meta-gen)
+    (alias misc)
+    (alias multiple-private-libs)
+    (alias null-dep)
+    (alias ocaml-syntax)
+    (alias ocamldep-multi-stanzas)
+    (alias output-obj)
+    (alias package-dep)
+    (alias promote)
+    (alias quoting)
+    (alias redirections)
+    (alias scope-bug)
+    (alias scope-ppx-bug)
+    (alias select)
+    (alias utop)))))
+
+(alias ((name runtest-disabled) (deps ((alias reason)))))
+
+(alias ((name runtest-js) (deps ((alias js_of_ocaml)))))
\ No newline at end of file
diff --git a/test/blackbox-tests/platform.ml b/test/blackbox-tests/platform.ml
new file mode 100644
index 00000000..477478dd
--- /dev/null
+++ b/test/blackbox-tests/platform.ml
@@ -0,0 +1,30 @@
+open Stdune
+
+type t = Win | Mac
+
+let to_string = function
+  | Win -> "win"
+  | Mac -> "macosx"
+
+let of_string = function
+  | "macosx" -> Mac
+  | "win" -> Win
+  | s -> invalid_arg ("Platform.of_string: " ^ s)
+
+let arg_name = "-skip-platforms"
+
+let to_cmd = function
+  | [] -> []
+  | l ->
+    [ arg_name
+    ; String.concat ~sep:"," (List.map ~f:to_string l)]
+
+let of_args args =
+  String.split ~on:',' args
+  |> List.map ~f:of_string
+
+let argv v =
+  ( arg_name
+  , Arg.String (fun s -> v := of_args s)
+  , "Comma separated versions of ocaml where to skip test"
+  )
diff --git a/test/blackbox-tests/test-cases/github25/root/run.t b/test/blackbox-tests/test-cases/github25/root/run.t
deleted file mode 100644
index 53ba3d05..00000000
--- a/test/blackbox-tests/test-cases/github25/root/run.t
+++ /dev/null
@@ -1,24 +0,0 @@
-This test define an installed "plop" with a "plop.ca-marche-pas"
-sub-package which depend on a library that doesn't exist.
-
-The build itself uses only "plop.ca-marche", which doesn't have this
-problem. So jbuilder shouldn't crash because of "plop.ca-marche-pas"
-
-We need ocamlfind to run this test
-
-  $ jbuilder build @install --display short --only hello
-        ocamlc .hello.objs/hello.{cmi,cmo,cmt}
-      ocamlopt .hello.objs/hello.{cmx,o}
-      ocamlopt hello.{a,cmxa}
-      ocamlopt hello.cmxs
-        ocamlc hello.cma
-
-  $ jbuilder build @install --display short --only pas-de-bol 2>&1 | sed 's/[^ "]*findlib-packages/.../'
-      ocamldep a.ml.d
-  File ".../plop/META", line 1, characters 0-0:
-  Error: Library "une-lib-qui-nexiste-pas" not found.
-  -> required by library "plop.ca-marche-pas" in .../plop
-  Hint: try: jbuilder external-lib-deps --missing --only-packages pas-de-bol @install
-      ocamldep b.ml.d
-        ocamlc .pas_de_bol.objs/pas_de_bol.{cmi,cmo,cmt}
-      ocamlopt .pas_de_bol.objs/pas_de_bol.{cmx,o}
diff --git a/test/blackbox-tests/test-cases/github25/run.t b/test/blackbox-tests/test-cases/github25/run.t
new file mode 100644
index 00000000..67f99513
--- /dev/null
+++ b/test/blackbox-tests/test-cases/github25/run.t
@@ -0,0 +1,27 @@
+This test define an installed "plop" with a "plop.ca-marche-pas"
+sub-package which depend on a library that doesn't exist.
+
+The build itself uses only "plop.ca-marche", which doesn't have this
+problem. So jbuilder shouldn't crash because of "plop.ca-marche-pas"
+
+We need ocamlfind to run this test
+
+  $ jbuilder build @install --display short --only hello
+  File "root/jbuild", line 5, characters 14-28:
+  Error: Library "plop.ca-marche" not found.
+  Hint: try: jbuilder external-lib-deps --missing --only-packages hello @install
+        ocamlc root/.hello.objs/hello.{cmi,cmo,cmt}
+      ocamlopt root/.hello.objs/hello.{cmx,o}
+      ocamlopt root/hello.{a,cmxa}
+      ocamlopt root/hello.cmxs
+        ocamlc root/hello.cma
+  [1]
+
+  $ jbuilder build @install --display short --only pas-de-bol 2>&1 | sed 's/[^ "]*findlib-packages/.../'
+  File "root/jbuild", line 11, characters 14-32:
+  Error: Library "plop.ca-marche-pas" not found.
+  Hint: try: jbuilder external-lib-deps --missing --only-packages pas-de-bol @install
+      ocamldep root/a.ml.d
+      ocamldep root/b.ml.d
+        ocamlc root/.pas_de_bol.objs/pas_de_bol.{cmi,cmo,cmt}
+      ocamlopt root/.pas_de_bol.objs/pas_de_bol.{cmx,o}