From 23576a4832e3b55a81c63c44d1f4b5eab516506b Mon Sep 17 00:00:00 2001 From: Anil Madhavapeddy Date: Fri, 1 Jun 2018 20:14:36 +0100 Subject: [PATCH 1/5] configurator: add write_lines function The `write_flags` only works with `(:include` directives, and it is also useful to be able to write a list of lines so that the discovered information can be used in variable expansion actions. For example, ocaml-yaml discovers CFLAGS and then directly has `(run ${CC} ${read-lines:cflags})` actions that use this new write_lines function to list cflags instead of s-expressions. They must be line-by-line or else variable expansion doesnt work since CFLAGS contain spaces. Signed-off-by: Anil Madhavapeddy --- src/configurator/v1.ml | 27 ++++++++++++++++++++------- src/configurator/v1.mli | 30 ++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/configurator/v1.ml b/src/configurator/v1.ml index 1f11d417..152022d9 100644 --- a/src/configurator/v1.ml +++ b/src/configurator/v1.ml @@ -61,6 +61,24 @@ module Temp = struct dir end +module Flags = struct + let extract_words = String.extract_words + + let extract_comma_space_separated_words = + String.extract_comma_space_separated_words + + let extract_blank_separated_words = String.extract_blank_separated_words + + let write_lines fname s = + let path = Path.of_string fname in + Io.write_lines path s + + let write_sexp fname s = + let path = Path.in_source fname in + let sexp = Usexp.List (List.map s ~f:(fun s -> Usexp.Quoted_string s)) in + Io.write_file path (Usexp.to_string sexp ~syntax:Dune) +end + module Find_in_path = struct let path_sep = if Sys.win32 then @@ -485,18 +503,13 @@ module Pkg_config = struct None end -let write_flags fname s = - let path = Path.in_source fname in - let sexp = Usexp.List (List.map s ~f:(fun s -> Usexp.Quoted_string s)) in - Io.write_file path (Usexp.to_string sexp ~syntax:Dune) - let main ?(args=[]) ~name f = let ocamlc = ref ( match Sys.getenv "DUNE_CONFIGURATOR" with | s -> Some s | exception Not_found -> - die "Configurator scripts must be ran with jbuilder. \ - To manually run a script, use $ jbuilder exec." + die "Configurator scripts must be run with Dune. \ + To manually run a script, use $ dune exec." ) in let verbose = ref false in let dest_dir = ref None in diff --git a/src/configurator/v1.mli b/src/configurator/v1.mli index 5eb9ebb7..20cc9292 100644 --- a/src/configurator/v1.mli +++ b/src/configurator/v1.mli @@ -87,10 +87,32 @@ module Pkg_config : sig val query : t -> package:string -> package_conf option end with type configurator := t -val write_flags : string -> string list -> unit -(** [write_flags fname s] write the list of strings [s] to the file - [fname] in an appropriate format so that it can used in jbuild - files with "(:include [fname])". *) +module Flags : sig + + val write_sexp : string -> string list -> unit + (** [write_sexp fname s] writes the list of strings [s] to the file [fname] in + an appropriate format so that it can used in jbuild files with [(:include + [fname])]. *) + + val write_lines : string -> string list -> unit + (** [write_lines fname s] writes the list of string [s] to the file [fname] + with one line per string so that it can be used in Dune action rules with + [%{read-lines:}]. *) + + val extract_comma_space_separated_words : string -> string list + (** [extract_comma_space_separated_words s] returns a list of words in + [s] that are separated by a newline, tab, space or comma character. *) + + val extract_blank_separated_words : string -> string list + (** [extract_blank_separated_words s] returns a list of words in [s] + that are separated by a tab or space character. *) + + val extract_words : string -> is_word_char:(char -> bool) -> string list + (** [extract_words s ~is_word_char] will split the string [s] into + a list of words. A valid word character is defined by the [is_word_char] + predicate returning true and anything else is considered a separator. + Any blank words are filtered out of the results. *) +end (** Typical entry point for configurator programs *) val main From 629b3d07ee8aa112cfa7214adf712fd0e584cdb1 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 9 Jul 2018 00:33:07 +0700 Subject: [PATCH 2/5] Add test to show bug with expanding %{ocaml:..} forms Signed-off-by: Rudi Grinberg --- test/blackbox-tests/dune.inc | 10 ++++++++++ test/blackbox-tests/test-cases/ocaml-config-macro/dune | 3 +++ .../test-cases/ocaml-config-macro/dune-project | 1 + .../blackbox-tests/test-cases/ocaml-config-macro/run.t | 7 +++++++ 4 files changed, 21 insertions(+) create mode 100644 test/blackbox-tests/test-cases/ocaml-config-macro/dune create mode 100644 test/blackbox-tests/test-cases/ocaml-config-macro/dune-project create mode 100644 test/blackbox-tests/test-cases/ocaml-config-macro/run.t diff --git a/test/blackbox-tests/dune.inc b/test/blackbox-tests/dune.inc index bf0c54a1..984ac606 100644 --- a/test/blackbox-tests/dune.inc +++ b/test/blackbox-tests/dune.inc @@ -436,6 +436,14 @@ test-cases/null-dep (progn (run %{exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))) +(alias + (name ocaml-config-macro) + (deps (package dune) (source_tree test-cases/ocaml-config-macro)) + (action + (chdir + test-cases/ocaml-config-macro + (progn (run %{exe:cram.exe} -test run.t) (diff? run.t run.t.corrected))))) + (alias (name ocaml-syntax) (deps (package dune) (source_tree test-cases/ocaml-syntax)) @@ -698,6 +706,7 @@ (alias multiple-private-libs) (alias no-installable-mode) (alias null-dep) + (alias ocaml-config-macro) (alias ocaml-syntax) (alias ocamldep-multi-stanzas) (alias odoc) @@ -773,6 +782,7 @@ (alias misc) (alias no-installable-mode) (alias null-dep) + (alias ocaml-config-macro) (alias ocaml-syntax) (alias ocamldep-multi-stanzas) (alias output-obj) diff --git a/test/blackbox-tests/test-cases/ocaml-config-macro/dune b/test/blackbox-tests/test-cases/ocaml-config-macro/dune new file mode 100644 index 00000000..55237b58 --- /dev/null +++ b/test/blackbox-tests/test-cases/ocaml-config-macro/dune @@ -0,0 +1,3 @@ +(rule + (targets x) + (action (with-stdout-to %{targets} (echo %{ocaml-config:system})))) \ No newline at end of file diff --git a/test/blackbox-tests/test-cases/ocaml-config-macro/dune-project b/test/blackbox-tests/test-cases/ocaml-config-macro/dune-project new file mode 100644 index 00000000..b2559fa0 --- /dev/null +++ b/test/blackbox-tests/test-cases/ocaml-config-macro/dune-project @@ -0,0 +1 @@ +(lang dune 1.0) \ No newline at end of file diff --git a/test/blackbox-tests/test-cases/ocaml-config-macro/run.t b/test/blackbox-tests/test-cases/ocaml-config-macro/run.t new file mode 100644 index 00000000..512709cd --- /dev/null +++ b/test/blackbox-tests/test-cases/ocaml-config-macro/run.t @@ -0,0 +1,7 @@ +%{ocaml-config:...} macros should be available. we don't print anything because +the values are all platform specific. + + $ dune build + File "dune", line 3, characters 41-61: + Error: Unknown form: %{ocaml-config:system} + [1] \ No newline at end of file From 0ca157f8400fdb848b434363e99d8d65d9463dec Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 9 Jul 2018 00:50:42 +0700 Subject: [PATCH 3/5] Fix expansion of %{ocaml-config:..} Signed-off-by: Rudi Grinberg --- src/pform.ml | 15 +++------------ src/pform.mli | 1 + src/super_context.ml | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/pform.ml b/src/pform.ml index f975473b..58e1858b 100644 --- a/src/pform.ml +++ b/src/pform.ml @@ -30,6 +30,7 @@ module Macro = struct | Read_strings | Read_lines | Path_no_dep + | Ocaml_config end type 'a t = @@ -77,6 +78,7 @@ module Map = struct ; "findlib", renamed_in ~version:(1, 0) ~new_name:"lib" ; "path-no-dep", deleted_in ~version:(1, 0) Path_no_dep + ; "ocaml-config", macro Ocaml_config ] |> String.Map.of_list_exn @@ -126,18 +128,7 @@ module Map = struct ; "profile" , string context.profile ] in - let ocaml_config = - List.map (Ocaml_config.to_list context.ocaml_config) ~f:(fun (k, v) -> - ("ocaml-config:" ^ k, - match (v : Ocaml_config.Value.t) with - | Bool x -> string (string_of_bool x) - | Int x -> string (string_of_int x) - | String x -> string x - | Words x -> strings x - | Prog_and_args x -> strings (x.prog :: x.args))) - in - [ ocaml_config - ; static_vars + [ static_vars ; lowercased ; uppercased ; vars diff --git a/src/pform.mli b/src/pform.mli index d36d3624..357a3a16 100644 --- a/src/pform.mli +++ b/src/pform.mli @@ -22,6 +22,7 @@ module Macro : sig | Read_strings | Read_lines | Path_no_dep + | Ocaml_config end type 'a t = diff --git a/src/super_context.ml b/src/super_context.ml index 21b3fe09..46c9c865 100644 --- a/src/super_context.ml +++ b/src/super_context.ml @@ -47,6 +47,7 @@ type t = ; cxx_flags : string list ; vars : Pform.Var.t Pform.Map.t ; macros : Pform.Macro.t Pform.Map.t + ; ocaml_config : Value.t list String.Map.t ; chdir : (Action.t, Action.t) Build.t ; host : t option ; libs_by_package : (Package.t * Lib.Set.t) Package.Name.Map.t @@ -290,6 +291,19 @@ let create ~f:(fun s -> not (String.is_prefix s ~prefix:"-std=")) in let vars = Pform.Map.create_vars ~context ~cxx_flags in + let ocaml_config = + let string s = [Value.String s] in + Ocaml_config.to_list context.ocaml_config + |> List.map ~f:(fun (k, v) -> + ( k + , match (v : Ocaml_config.Value.t) with + | Bool x -> string (string_of_bool x) + | Int x -> string (string_of_int x) + | String x -> string x + | Words x -> Value.L.strings x + | Prog_and_args x -> Value.L.strings (x.prog :: x.args))) + |> String.Map.of_list_exn + in let t = { context ; host @@ -305,6 +319,7 @@ let create ; cxx_flags ; vars ; macros = Pform.Map.macros + ; ocaml_config ; chdir = Build.arr (fun (action : Action.t) -> match action with | Chdir _ -> action @@ -597,6 +612,7 @@ module Action = struct let key = String_with_vars.Var.full_name var in begin match expand_macro sctx ~syntax_version ~var with | Some Pform.Macro.Exe -> Some (path_exp (map_exe (Path.relative dir s))) + | Some Ocaml_config -> String.Map.find sctx.ocaml_config s | Some Dep -> Some (path_exp (Path.relative dir s)) | Some Bin -> begin let sctx = host sctx in From baf492db119ddf16c8e05759d304ffd148bd0508 Mon Sep 17 00:00:00 2001 From: Rudi Grinberg Date: Mon, 9 Jul 2018 01:00:11 +0700 Subject: [PATCH 4/5] Allow %{ocaml-config:..} wherever variables are allowed Signed-off-by: Rudi Grinberg --- src/super_context.ml | 22 ++++++++++++++++--- .../test-cases/macro-expand-error/run.t | 2 +- .../test-cases/ocaml-config-macro/run.t | 3 --- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/super_context.ml b/src/super_context.ml index 46c9c865..00a797ce 100644 --- a/src/super_context.ml +++ b/src/super_context.ml @@ -100,20 +100,36 @@ let expand_macro t ~syntax_version ~var = Exn.code_error "expand_macro can't expand variables" [ "var", String_with_vars.Var.sexp_of_t var ] +let expand t ~syntax_version ~var = + match + match String_with_vars.Var.destruct var with + | Var _ -> Left (expand_vars t ~syntax_version ~var) + | Macro (_, _) -> Right (expand_macro t ~syntax_version ~var) + with + | Right None + | Left None -> None + | Right (Some x) -> Some (Right x) + | Left (Some x) -> Some (Left x) + let (expand_vars_string, expand_vars_path) = let expand t ~scope ~dir ?(extra_vars=String.Map.empty) s = String_with_vars.expand ~mode:Single ~dir s ~f:(fun var syntax_version -> - match expand_vars t ~syntax_version ~var with + match expand t ~syntax_version ~var with | None -> String.Map.find extra_vars (String_with_vars.Var.full_name var) - | Some v -> + | Some (Left v) -> begin match Pform.Var.to_value_no_deps_or_targets ~scope v with | Some _ as v -> v | None -> Loc.fail (String_with_vars.Var.loc var) "Variable %a is not allowed in this context" String_with_vars.Var.pp var - end) + end + | Some (Right Ocaml_config) -> + String.Map.find t.ocaml_config (String_with_vars.Var.name var) + | Some (Right _) -> + Loc.fail (String_with_vars.Var.loc var) + "This percent form isn't allowed in this position") in let expand_vars t ~scope ~dir ?extra_vars s = expand t ~scope ~dir ?extra_vars s diff --git a/test/blackbox-tests/test-cases/macro-expand-error/run.t b/test/blackbox-tests/test-cases/macro-expand-error/run.t index 832d5db8..99d53da5 100644 --- a/test/blackbox-tests/test-cases/macro-expand-error/run.t +++ b/test/blackbox-tests/test-cases/macro-expand-error/run.t @@ -4,5 +4,5 @@ inappropariate place: $ dune build Info: creating file dune-project with this contents: (lang dune 1.0) File "dune", line 1, characters 14-21: - Error: macros of the form %{name:..} cannot be expanded here + Error: This percent form isn't allowed in this position [1] diff --git a/test/blackbox-tests/test-cases/ocaml-config-macro/run.t b/test/blackbox-tests/test-cases/ocaml-config-macro/run.t index 512709cd..af1b2a3d 100644 --- a/test/blackbox-tests/test-cases/ocaml-config-macro/run.t +++ b/test/blackbox-tests/test-cases/ocaml-config-macro/run.t @@ -2,6 +2,3 @@ the values are all platform specific. $ dune build - File "dune", line 3, characters 41-61: - Error: Unknown form: %{ocaml-config:system} - [1] \ No newline at end of file From 0bf8140ba10ccc6b1145790344a60d2dc53f0d35 Mon Sep 17 00:00:00 2001 From: Jeremie Dimino Date: Mon, 9 Jul 2018 09:40:52 +0100 Subject: [PATCH 5/5] Fix output-obj tests Signed-off-by: Jeremie Dimino --- test/blackbox-tests/test-cases/output-obj/dune | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/blackbox-tests/test-cases/output-obj/dune b/test/blackbox-tests/test-cases/output-obj/dune index c4858bf5..591b5ab5 100644 --- a/test/blackbox-tests/test-cases/output-obj/dune +++ b/test/blackbox-tests/test-cases/output-obj/dune @@ -21,19 +21,19 @@ (rule (targets static.exe) (deps test.exe%{ext_obj} static.c) - (action (run %{CC} -o %{targets} -I %{ocaml_where} -I . %{deps} + (action (run %{cc} -o %{targets} -I %{ocaml_where} -I . %{deps} %{ocaml-config:native_c_libraries}))) (rule (targets static.bc) (deps test.bc%{ext_obj} static.c) - (action (run %{CC} -o %{targets} -I %{ocaml_where} -I . %{deps} + (action (run %{cc} -o %{targets} -I %{ocaml_where} -I . %{deps} %{ocaml-config:bytecomp_c_libraries}))) (rule (targets dynamic.exe) (deps dynamic.c) - (action (run %{CC} -o %{targets} %{first-dep} %{ocaml-config:native_c_libraries}))) + (action (run %{cc} -o %{targets} %{first-dep} %{ocaml-config:native_c_libraries}))) (alias (name runtest)