Expand variables in flags

Extend Ordered_set_lang.Unexpanded.expand to include a mapping function
for the S-expression for each atom. The previous behaviour can be
achieved with ~f:Sexp.Of_sexp.string, but this allows the S-expression
to be parsed using String_with_vars.t, thus allowing variable expansion.

src/gen_rules.ml

@@ -96,7 +96,7 @@ module Gen(P : Params) = struct
                 ~modules
                 ~mode
                 (String_map.keys modules)))
-           (SC.expand_and_eval_set ~dir lib.c_library_flags ~standard:[])
+           (SC.expand_and_eval_set sctx ~scope ~dir lib.c_library_flags ~standard:[])
            (Ocaml_flags.get flags mode)
          >>>
          Build.run ~context:ctx (Dep compiler)
@@ -115,14 +115,14 @@ module Gen(P : Params) = struct
            ; Dyn (fun (cm_files, _, _) -> Deps cm_files)
            ]))
 
-  let build_c_file (lib : Library.t) ~dir ~requires ~h_files c_name =
+  let build_c_file (lib : Library.t) ~scope ~dir ~requires ~h_files c_name =
     let src = Path.relative dir (c_name ^ ".c") in
     let dst = Path.relative dir (c_name ^ ctx.ext_obj) in
     SC.add_rule sctx
       (Build.paths h_files
        >>>
        Build.fanout
-         (SC.expand_and_eval_set ~dir lib.c_flags ~standard:(Context.cc_g ctx))
+         (SC.expand_and_eval_set sctx ~scope ~dir lib.c_flags ~standard:(Context.cc_g ctx))
          requires
        >>>
        Build.run ~context:ctx
@@ -140,14 +140,14 @@ module Gen(P : Params) = struct
          ]);
     dst
 
-  let build_cxx_file (lib : Library.t) ~dir ~requires ~h_files c_name =
+  let build_cxx_file (lib : Library.t) ~scope ~dir ~requires ~h_files c_name =
     let src = Path.relative dir (c_name ^ ".cpp") in
     let dst = Path.relative dir (c_name ^ ctx.ext_obj) in
     SC.add_rule sctx
       (Build.paths h_files
        >>>
        Build.fanout
-         (SC.expand_and_eval_set ~dir lib.cxx_flags ~standard:(Context.cc_g ctx))
+         (SC.expand_and_eval_set sctx ~scope ~dir lib.cxx_flags ~standard:(Context.cc_g ctx))
          requires
        >>>
        Build.run ~context:ctx
@@ -179,7 +179,7 @@ module Gen(P : Params) = struct
 
   let library_rules (lib : Library.t) ~dir ~all_modules ~files ~scope =
     let dep_kind = if lib.optional then Build.Optional else Required in
-    let flags = Ocaml_flags.make lib.buildable ~dir in
+    let flags = Ocaml_flags.make lib.buildable sctx ~scope ~dir in
     let modules =
       parse_modules ~dir ~all_modules ~modules_written_by_user:lib.buildable.modules
     in
@@ -269,7 +269,7 @@ module Gen(P : Params) = struct
     let dynlink = lib.dynlink in
     let js_of_ocaml = lib.buildable.js_of_ocaml in
     Module_compilation.build_modules sctx
-      ~js_of_ocaml ~dynlink ~flags ~dir ~dep_graph ~modules ~requires ~alias_module;
+      ~js_of_ocaml ~dynlink ~flags ~scope ~dir ~dep_graph ~modules ~requires ~alias_module;
     Option.iter alias_module ~f:(fun m ->
       let flags = Ocaml_flags.default () in
       Module_compilation.build_module sctx m
@@ -277,6 +277,7 @@ module Gen(P : Params) = struct
         ~dynlink
         ~sandbox:alias_module_build_sandbox
         ~flags:(Ocaml_flags.append_common flags ["-w"; "-49"])
+        ~scope
         ~dir
         ~modules:(String_map.singleton m.name m)
         ~dep_graph:(Ml_kind.Dict.make_both (Build.return (String_map.singleton m.name [])))
@@ -305,15 +306,15 @@ module Gen(P : Params) = struct
           Build.memoize "header files"
             (requires >>> SC.Libs.file_deps sctx ~ext:".h")
         in
-        List.map lib.c_names   ~f:(build_c_file   lib ~dir ~requires ~h_files) @
+        List.map lib.c_names   ~f:(build_c_file   lib ~scope ~dir ~requires ~h_files) @
-        List.map lib.cxx_names ~f:(build_cxx_file lib ~dir ~requires ~h_files)
+        List.map lib.cxx_names ~f:(build_cxx_file lib ~scope ~dir ~requires ~h_files)
       in
       match lib.self_build_stubs_archive with
       | Some _ -> ()
       | None ->
         let ocamlmklib ~sandbox ~custom ~targets =
           SC.add_rule sctx ~sandbox
-            (SC.expand_and_eval_set ~dir lib.c_library_flags ~standard:[]
+            (SC.expand_and_eval_set sctx ~scope ~dir lib.c_library_flags ~standard:[]
              >>>
              Build.run ~context:ctx
                ~extra_targets:targets
@@ -367,7 +368,7 @@ module Gen(P : Params) = struct
     (* Build *.cma.js *)
     SC.add_rules sctx (
       let src = lib_archive lib ~dir ~ext:(Mode.compiled_lib_ext Mode.Byte) in
-      Js_of_ocaml_rules.build_cm sctx ~dir ~js_of_ocaml:lib.buildable.js_of_ocaml ~src);
+      Js_of_ocaml_rules.build_cm sctx ~scope ~dir ~js_of_ocaml:lib.buildable.js_of_ocaml ~src);
 
     if ctx.natdynlink_supported then
       Option.iter ctx.ocamlopt ~f:(fun ocamlopt ->
@@ -415,7 +416,7 @@ module Gen(P : Params) = struct
      | Executables stuff                                               |
      +-----------------------------------------------------------------+ *)
 
-  let build_exe ~js_of_ocaml ~flags ~dir ~requires ~name ~mode ~modules ~dep_graph
+  let build_exe ~js_of_ocaml ~flags ~scope ~dir ~requires ~name ~mode ~modules ~dep_graph
         ~link_flags ~force_custom_bytecode =
     let exe_ext = Mode.exe_ext mode in
     let mode, link_custom, compiler =
@@ -443,7 +444,7 @@ module Gen(P : Params) = struct
        &&&
        Build.fanout
        (Ocaml_flags.get flags mode)
-       (SC.expand_and_eval_set ~dir link_flags ~standard:[])
+       (SC.expand_and_eval_set sctx ~scope ~dir link_flags ~standard:[])
        >>>
        Build.run ~context:ctx
          (Dep compiler)
@@ -458,13 +459,13 @@ module Gen(P : Params) = struct
       let libs_and_cm_and_flags =
         libs_and_cm
         &&&
-        SC.expand_and_eval_set ~dir js_of_ocaml.flags ~standard:(Js_of_ocaml_rules.standard ())
+        SC.expand_and_eval_set sctx ~scope ~dir js_of_ocaml.flags ~standard:(Js_of_ocaml_rules.standard ())
       in
       SC.add_rules sctx (List.map rules ~f:(fun r -> libs_and_cm_and_flags >>> r))
 
   let executables_rules (exes : Executables.t) ~dir ~all_modules ~scope =
     let dep_kind = Build.Required in
-    let flags = Ocaml_flags.make exes.buildable ~dir in
+    let flags = Ocaml_flags.make exes.buildable sctx ~scope ~dir in
     let modules =
       parse_modules ~dir ~all_modules ~modules_written_by_user:exes.buildable.modules
     in
@@ -501,12 +502,12 @@ module Gen(P : Params) = struct
     (* CR-someday jdimino: this should probably say [~dynlink:false] *)
     Module_compilation.build_modules sctx
       ~js_of_ocaml:exes.buildable.js_of_ocaml
-      ~dynlink:true ~flags ~dir ~dep_graph ~modules
+      ~dynlink:true ~flags ~scope ~dir ~dep_graph ~modules
       ~requires ~alias_module:None;
 
     List.iter exes.names ~f:(fun name ->
       List.iter Mode.all ~f:(fun mode ->
-        build_exe ~js_of_ocaml:exes.buildable.js_of_ocaml ~flags ~dir ~requires ~name
+        build_exe ~js_of_ocaml:exes.buildable.js_of_ocaml ~flags ~scope ~dir ~requires ~name
           ~mode ~modules ~dep_graph ~link_flags:exes.link_flags
           ~force_custom_bytecode:(mode = Native && not exes.modes.native)));
     { Merlin.

src/js_of_ocaml_rules.ml

@@ -89,12 +89,12 @@ let link_rule ~sctx ~dir ~runtime ~target =
     ; Arg_spec.Dyn get_all
     ]
 
-let build_cm sctx ~dir ~js_of_ocaml ~src =
+let build_cm sctx ~scope ~dir ~js_of_ocaml ~src =
   if separate_compilation_enabled ()
   then let target = Path.extend_basename src ~suffix:".js" in
     let spec = Arg_spec.Dep src in
     let flags =
-      SC.expand_and_eval_set ~dir js_of_ocaml.Jbuild.Js_of_ocaml.flags
+      SC.expand_and_eval_set sctx ~scope ~dir js_of_ocaml.Jbuild.Js_of_ocaml.flags
         ~standard:(standard ())
     in
     [ flags

src/js_of_ocaml_rules.mli

@@ -4,6 +4,7 @@ open Jbuild
 
 val build_cm
   :  Super_context.t
+  -> scope:Scope.t
   -> dir:Path.t
   -> js_of_ocaml:Js_of_ocaml.t
   -> src:Path.t

src/module_compilation.ml

@@ -79,7 +79,7 @@ let build_cm sctx ?sandbox ~dynlink ~flags ~cm_kind ~(dep_graph:Ocamldep.dep_gra
            ; A "-c"; Ml_kind.flag ml_kind; Dep src
            ])))
 
-let build_module sctx ?sandbox ~dynlink ~js_of_ocaml ~flags m ~dir ~dep_graph
+let build_module sctx ?sandbox ~dynlink ~js_of_ocaml ~flags m ~scope ~dir ~dep_graph
       ~modules ~requires ~alias_module =
   List.iter Cm_kind.all ~f:(fun cm_kind ->
     let requires = Cm_kind.Dict.get requires cm_kind in
@@ -87,9 +87,9 @@ let build_module sctx ?sandbox ~dynlink ~js_of_ocaml ~flags m ~dir ~dep_graph
       ~requires ~alias_module);
   (* Build *.cmo.js *)
   let src = Module.cm_file m ~dir Cm_kind.Cmo in
-  SC.add_rules sctx (Js_of_ocaml_rules.build_cm sctx ~dir ~js_of_ocaml ~src)
+  SC.add_rules sctx (Js_of_ocaml_rules.build_cm sctx ~scope ~dir ~js_of_ocaml ~src)
 
-let build_modules sctx ~dynlink ~js_of_ocaml ~flags ~dir ~dep_graph ~modules ~requires
+let build_modules sctx ~dynlink ~js_of_ocaml ~flags ~scope ~dir ~dep_graph ~modules ~requires
       ~alias_module =
   let cmi_requires =
     Build.memoize "cmi library dependencies"
@@ -114,5 +114,5 @@ let build_modules sctx ~dynlink ~js_of_ocaml ~flags ~dir ~dep_graph ~modules ~re
      | None -> modules
      | Some (m : Module.t) -> String_map.remove m.name modules)
     ~f:(fun ~key:_ ~data:m ->
-      build_module sctx m ~dynlink ~js_of_ocaml ~flags ~dir ~dep_graph ~modules ~requires
+      build_module sctx m ~dynlink ~js_of_ocaml ~flags ~scope ~dir ~dep_graph ~modules ~requires
         ~alias_module)

src/module_compilation.mli

@@ -13,6 +13,7 @@ val build_module
   -> js_of_ocaml:Jbuild.Js_of_ocaml.t
   -> flags:Ocaml_flags.t
   -> Module.t
+  -> scope:Jbuild.Scope.t
   -> dir:Path.t
   -> dep_graph:Ocamldep.dep_graph
   -> modules:Module.t String_map.t
@@ -26,6 +27,7 @@ val build_modules
   -> dynlink:bool
   -> js_of_ocaml:Jbuild.Js_of_ocaml.t
   -> flags:Ocaml_flags.t
+  -> scope:Jbuild.Scope.t
   -> dir:Path.t
   -> dep_graph:Ocamldep.dep_graph
   -> modules:Module.t String_map.t

src/ocaml_flags.ml

@@ -37,8 +37,8 @@ type t =
   ; specific   : (unit, string list) Build.t Mode.Dict.t
   }
 
-let make { Jbuild.Buildable. flags; ocamlc_flags; ocamlopt_flags; _ } ~dir =
+let make { Jbuild.Buildable. flags; ocamlc_flags; ocamlopt_flags; _ } ctx ~scope ~dir =
-  let eval = Super_context.expand_and_eval_set ~dir in
+  let eval = Super_context.expand_and_eval_set ctx ~scope ~dir in
   { common   = Build.memoize "common flags" (eval flags ~standard:(default_flags ()))
   ; specific =
       { byte   = Build.memoize "ocamlc flags" (eval ocamlc_flags   ~standard:(default_ocamlc_flags ()))

src/ocaml_flags.mli

@@ -2,7 +2,7 @@
 
 type t
 
-val make : Jbuild.Buildable.t -> dir:Path.t -> t
+val make : Jbuild.Buildable.t -> Super_context.t -> scope:Jbuild.Scope.t -> dir:Path.t -> t
 
 val default : unit -> t
 

src/ordered_set_lang.ml

@@ -9,20 +9,20 @@ module Ast = struct
     | Special : Loc.t * string -> ('a, _) t
     | Union : ('a, 'b) t list -> ('a, 'b) t
     | Diff : ('a, 'b) t * ('a, 'b) t -> ('a, 'b) t
-    | Include : 'a -> ('a, unexpanded) t
+    | Include : string -> ('a, unexpanded) t
 end
 
 type t = (string, Ast.expanded) Ast.t
 
-let t t : t =
+let parse_general t ~f =
   let rec of_sexp : Sexp.Ast.t -> _ = function
     | Atom (loc, "\\") -> Loc.fail loc "unexpected \\"
-    | Atom (_, "") -> Ast.Element ""
+    | Atom (_, "") as t -> Ast.Element (f t)
-    | Atom (loc, s) ->
+    | Atom (loc, s) as t ->
       if s.[0] = ':' then
         Special (loc, String.sub s ~pos:1 ~len:(String.length s - 1))
       else
-        Element s
+        Element (f t)
     | List (_, sexps) -> of_sexps [] sexps
   and of_sexps acc = function
     | Atom (_, "\\") :: sexps -> Diff (Union (List.rev acc), of_sexps [] sexps)
@@ -32,6 +32,8 @@ let t t : t =
   in
   of_sexp t
 
+let t t : t = parse_general t ~f:(function Atom (_, s) -> s | List _ -> assert false)
+
 let eval t ~special_values =
   let rec of_ast (t : t) =
     let open Ast in
@@ -74,22 +76,21 @@ let standard = Ast.Special (Loc.none, "standard")
 let append a b = Ast.Union [a; b]
 
 module Unexpanded = struct
-  type t = (string, Ast.unexpanded) Ast.t
+  type t = (Sexp.Ast.t, Ast.unexpanded) Ast.t
-  let parse_expanded = t
+  let t t =
-  let t t' =
+    let rec map (t : (Sexp.Ast.t, Ast.expanded) Ast.t) =
-    let rec map (t : (string, Ast.expanded) Ast.t) =
       let open Ast in
       match t with
       | Element s -> Element s
       | Special (l, s) -> Special (l, s)
       | Union [Special (_, "include"); Element fn] ->
-          Include fn
+          Include (Sexp.Of_sexp.string fn)
       | Union l ->
           Union (List.map l ~f:map)
       | Diff (l, r) ->
           Diff (map l, map r)
     in
-    t t' |> map
+    parse_general t ~f:(fun x -> x) |> map
 
   let standard = standard
 
@@ -110,16 +111,16 @@ module Unexpanded = struct
     in
     loop String_set.empty t
 
-  let rec expand (t : t) ~files_contents : (string, Ast.expanded) Ast.t =
+  let rec expand (t : t) ~files_contents ~f : (string, Ast.expanded) Ast.t =
     let open Ast in
     match t with
-    | Element s -> Element s
+    | Element s -> Element (f s)
     | Special (l, s) -> Special (l, s)
     | Include fn ->
-        parse_expanded (String_map.find_exn fn files_contents ~string_of_key:(sprintf "%S")
+        parse_general (String_map.find_exn fn files_contents ~string_of_key:(sprintf "%S")
-          ~desc:(fun _ -> "<filename to s-expression>"))
+          ~desc:(fun _ -> "<filename to s-expression>")) ~f
     | Union l ->
-        Union (List.map l ~f:(expand ~files_contents))
+        Union (List.map l ~f:(expand ~files_contents ~f))
     | Diff (l, r) ->
-        Diff (expand l ~files_contents, expand r ~files_contents)
+        Diff (expand l ~files_contents ~f, expand r ~files_contents ~f)
 end

src/ordered_set_lang.mli

@@ -27,6 +27,6 @@ module Unexpanded : sig
 
   (** Expand [t] using with the given file contents. [file_contents] is a map from
       filenames to their parsed contents. Every [(:include fn)] in [t] is replaced by
-      [Map.find files_contents fn]. *)
+      [Map.find files_contents fn]. Every element is converted to a string using [f]. *)
-  val expand : t -> files_contents:Sexp.Ast.t String_map.t -> expanded
+  val expand : t -> files_contents:Sexp.Ast.t String_map.t -> f:(Sexp.Ast.t -> string) -> expanded
 end with type expanded := t

src/super_context.ml

@@ -895,16 +895,17 @@ module PP = struct
     )
 end
 
-let expand_and_eval_set ~dir set ~standard =
+let expand_and_eval_set t ~scope ~dir set ~standard =
   let open Build.O in
+  let f sexp = expand_vars t ~scope ~dir (String_with_vars.t sexp) in
   match Ordered_set_lang.Unexpanded.files set |> String_set.elements with
   | [] ->
-    let set = Ordered_set_lang.Unexpanded.expand set ~files_contents:String_map.empty in
+    let set = Ordered_set_lang.Unexpanded.expand set ~files_contents:String_map.empty ~f in
     Build.return (Ordered_set_lang.eval_with_standard set ~standard)
   | files ->
     let paths = List.map files ~f:(Path.relative dir) in
     Build.all (List.map paths ~f:Build.read_sexp)
     >>^ fun sexps ->
     let files_contents = List.combine files sexps |> String_map.of_alist_exn in
-    let set = Ordered_set_lang.Unexpanded.expand set ~files_contents in
+    let set = Ordered_set_lang.Unexpanded.expand set ~files_contents ~f in
     Ordered_set_lang.eval_with_standard set ~standard

src/super_context.mli

@@ -168,7 +168,9 @@ module PP : sig
 end
 
 val expand_and_eval_set
-  :  dir:Path.t
+  :  t
+  -> scope:Scope.t
+  -> dir:Path.t
   -> Ordered_set_lang.Unexpanded.t
   -> standard:string list
   -> (unit, string list) Build.t