Split calls to ocamldep (#486)

Instead of doing a single call to ocamldep, do one per file. This is needed to support "menhir --infer". This should also make compilation go further when there are files with syntax errors.
2018-02-06 11:48:04 +00:00 · 2018-02-06 11:48:04 +00:00 · 80c0bfc879
parent a1c02df143
commit 80c0bfc879
30 changed files with 269 additions and 256 deletions
--- a/CHANGES.md
+++ b/CHANGES.md
@ -1,3 +1,10 @@
+next
+----
+
+- Split calls to ocamldep. Before ocamldep would be called once per
+  `library`/`executables` stanza. Now it is called once per file
+  (#486)
+
 1.0+beta17 (01/02/2018)
 -----------------------

--- a/src/gen_rules.ml
+++ b/src/gen_rules.ml
@ -333,10 +333,9 @@ Add it to your jbuild file to remove this warning.
    in
    List.map cclibs ~f

-  let build_lib (lib : Library.t) ~scope ~flags ~dir ~mode ~modules ~dep_graph =
+  let build_lib (lib : Library.t) ~scope ~flags ~dir ~mode ~top_sorted_modules =
    Option.iter (Context.compiler ctx mode) ~f:(fun compiler ->
      let target = lib_archive lib ~dir ~ext:(Mode.compiled_lib_ext mode) in
-      let dep_graph = Ml_kind.Dict.get dep_graph Impl in
      let stubs_flags =
        if not (Library.has_stubs lib) then
          []
@ -361,14 +360,8 @@ Add it to your jbuild file to remove this warning.
      in
      SC.add_rule sctx
        (Build.fanout4
-           (dep_graph >>>
-            Build.arr (fun dep_graph ->
-              Ocamldep.names_to_top_closed_cm_files
-                ~dir
-                ~dep_graph
-                ~modules
-                ~mode
-                (String_map.keys modules)))
+           (top_sorted_modules >>^ List.map ~f:(fun m ->
+              Module.cm_file m ~dir (Mode.cm_kind mode)))
           (SC.expand_and_eval_set sctx ~scope ~dir lib.c_library_flags ~standard:[])
           (Ocaml_flags.get flags mode)
           (SC.expand_and_eval_set sctx ~scope ~dir lib.library_flags ~standard:[])
@ -474,8 +467,8 @@ Add it to your jbuild file to remove this warning.
      | Some m -> String_map.add modules ~key:m.name ~data:m
    in

-    let dep_graph =
-      Ocamldep.rules sctx ~dir ~item:lib.name ~modules ~alias_module
+    let dep_graphs =
+      Ocamldep.rules sctx ~dir ~modules ~alias_module
        ~lib_interface_module:(if lib.wrapped then
                                 String_map.find main_module_name modules
                               else
@ -510,7 +503,8 @@ Add it to your jbuild file to remove this warning.
    let dynlink = lib.dynlink in
    let js_of_ocaml = lib.buildable.js_of_ocaml in
    Module_compilation.build_modules sctx
-      ~js_of_ocaml ~dynlink ~flags ~scope:scope.data ~dir ~dep_graph ~modules ~requires ~alias_module;
+      ~js_of_ocaml ~dynlink ~flags ~scope:scope.data ~dir ~dep_graphs
+      ~modules ~requires ~alias_module;
    Option.iter alias_module ~f:(fun m ->
      let flags = Ocaml_flags.default () in
      Module_compilation.build_module sctx m
@ -520,8 +514,7 @@ Add it to your jbuild file to remove this warning.
        ~flags:(Ocaml_flags.append_common flags ["-w"; "-49"])
        ~scope:scope.data
        ~dir
-        ~modules:(String_map.singleton m.name m)
-        ~dep_graph:(Ml_kind.Dict.make_both (Build.return (String_map.singleton m.name [])))
+        ~dep_graphs:(Ocamldep.Dep_graphs.dummy m)
        ~requires:(
          let requires =
            if String_map.is_empty modules then
@ -608,8 +601,12 @@ Add it to your jbuild file to remove this warning.
      (List.map lib.install_c_headers ~f:(fun header ->
         Path.relative dir (header ^ ".h")));

+    let top_sorted_modules =
+      Build.memoize "top sorted modules" (
+        Ocamldep.Dep_graph.top_closed dep_graphs.impl (String_map.values modules))
+    in
    List.iter Mode.all ~f:(fun mode ->
-      build_lib lib ~scope:scope.data ~flags ~dir ~mode ~modules ~dep_graph);
+      build_lib lib ~scope:scope.data ~flags ~dir ~mode ~top_sorted_modules);
    (* Build *.cma.js *)
    SC.add_rules sctx (
      let src = lib_archive lib ~dir ~ext:(Mode.compiled_lib_ext Mode.Byte) in
@ -650,7 +647,7 @@ Add it to your jbuild file to remove this warning.
      String_set.fold files ~init:[] ~f:(fun fn acc ->
        if Filename.check_suffix fn ".mld" then fn :: acc else acc)
    in
-    Odoc.setup_library_rules sctx lib ~dir ~requires ~modules ~dep_graph
+    Odoc.setup_library_rules sctx lib ~dir ~requires ~modules ~dep_graphs
      ~mld_files
    ;

@ -671,28 +668,21 @@ Add it to your jbuild file to remove this warning.
     | Executables stuff                                               |
     +-----------------------------------------------------------------+ *)

-  let build_exe ~js_of_ocaml ~flags ~scope ~dir ~requires ~name ~mode ~modules ~dep_graph
-        ~link_flags ~force_custom_bytecode =
+  let build_exe ~js_of_ocaml ~flags ~scope ~dir ~requires ~name ~mode
+        ~top_sorted_modules ~link_flags ~force_custom_bytecode =
    let exe_ext = Mode.exe_ext mode in
    let mode, link_custom, compiler =
      match force_custom_bytecode, Context.compiler ctx mode with
      | false, Some compiler -> (mode, [], compiler)
      | _                    -> (Byte, ["-custom"], ctx.ocamlc)
    in
-    let dep_graph = Ml_kind.Dict.get dep_graph Impl in
    let exe = Path.relative dir (name ^ exe_ext) in
    let libs_and_cm =
      Build.fanout
        (requires
         >>> Build.dyn_paths (Build.arr (Lib.archive_files ~mode ~ext_lib:ctx.ext_lib)))
-        (dep_graph
-         >>> Build.arr (fun dep_graph ->
-           Ocamldep.names_to_top_closed_cm_files
-             ~dir
-             ~dep_graph
-             ~modules
-             ~mode
-             [String.capitalize_ascii name]))
+        (top_sorted_modules >>^ List.map ~f:(fun m ->
+           Module.cm_file m ~dir (Mode.cm_kind mode)))
    in
    let objs (libs, cm) =
      if mode = Mode.Byte then
@ -741,10 +731,14 @@ Add it to your jbuild file to remove this warning.
      String_map.map modules ~f:(fun (m : Module.t) ->
        { m with obj_name = Utils.obj_name_of_basename m.impl.name })
    in
-    List.iter exes.names ~f:(fun name ->
-      if not (String_map.mem (String.capitalize_ascii name) modules) then
-        die "executable %s in %s doesn't have a corresponding .ml file"
-          name (Path.to_string dir));
+    let programs =
+      List.map exes.names ~f:(fun name ->
+        match String_map.find (String.capitalize_ascii name) modules with
+        | Some m -> (name, m)
+        | None ->
+          die "executable %s in %s doesn't have a corresponding .ml file"
+            name (Path.to_string dir))
+    in

    let modules =
      SC.PP.pp_and_lint_modules sctx ~dir ~dep_kind ~modules ~scope
@ -755,8 +749,8 @@ Add it to your jbuild file to remove this warning.
    in

    let item = List.hd exes.names in
-    let dep_graph =
-      Ocamldep.rules sctx ~dir ~item ~modules ~alias_module:None
+    let dep_graphs =
+      Ocamldep.rules sctx ~dir ~modules ~alias_module:None
        ~lib_interface_module:None
    in

@ -773,13 +767,18 @@ Add it to your jbuild file to remove this warning.
    (* CR-someday jdimino: this should probably say [~dynlink:false] *)
    Module_compilation.build_modules sctx
      ~js_of_ocaml:exes.buildable.js_of_ocaml
-      ~dynlink:true ~flags ~scope:scope.data ~dir ~dep_graph ~modules
+      ~dynlink:true ~flags ~scope:scope.data ~dir ~dep_graphs ~modules
      ~requires ~alias_module:None;

-    List.iter exes.names ~f:(fun name ->
+    List.iter programs ~f:(fun (name, unit) ->
+      let top_sorted_modules =
+        Build.memoize "top sorted modules"
+          (Ocamldep.Dep_graph.top_closed dep_graphs.impl [unit])
+      in
      List.iter Mode.all ~f:(fun mode ->
        build_exe ~js_of_ocaml:exes.buildable.js_of_ocaml ~flags ~scope:scope.data
-          ~dir ~requires ~name ~mode ~modules ~dep_graph ~link_flags:exes.link_flags
+          ~dir ~requires ~name ~mode ~top_sorted_modules
+          ~link_flags:exes.link_flags
          ~force_custom_bytecode:(mode = Native && not exes.modes.native)));
    { Merlin.
      requires   = real_requires
--- a/src/import.ml
+++ b/src/import.ml
@ -150,12 +150,6 @@ module Map = struct
    val add_multi : 'a list t -> key:key -> data:'a -> 'a list t
    val find : key -> 'a t -> 'a option
    val find_default : key -> 'a t -> default:'a -> 'a
-    val find_exn
-      :  key
-      -> 'a t
-      -> string_of_key:(key -> string)
-      -> desc:('a t -> string)
-      -> 'a
    val of_alist : (key * 'a) list -> ('a t, key * 'a * 'a) result
    val of_alist_exn : (key * 'a) list -> 'a t
    val of_alist_multi : (key * 'a) list -> 'a list t
@ -216,13 +210,6 @@ module Map = struct

    let keys   t = bindings t |> List.map ~f:fst
    let values t = bindings t |> List.map ~f:snd
-
-    let find_exn key t ~string_of_key ~desc =
-      try
-        find_exn key t
-      with Not_found ->
-        code_errorf "%s not found in map %s"
-          (string_of_key key) (desc t)
  end
 end

--- a/src/module.ml
+++ b/src/module.ml
@ -32,6 +32,8 @@ type t =
  ; obj_name : string
  }

+let name t = t.name
+
 let real_unit_name t = String.capitalize_ascii (Filename.basename t.obj_name)

 let file t ~dir (kind : Ml_kind.t) =
--- a/src/module.mli
+++ b/src/module.mli
@ -23,6 +23,8 @@ type t =
                           modules. *)
  }

+val name : t -> string
+
 (** Real unit name once wrapped. This is always a valid module name. *)
 val real_unit_name : t -> string

--- a/src/module_compilation.ml
+++ b/src/module_compilation.ml
@ -4,8 +4,8 @@ open! No_io

 module SC = Super_context

-let build_cm sctx ?sandbox ~dynlink ~flags ~cm_kind ~(dep_graph:Ocamldep.dep_graph)
-      ~requires ~(modules : Module.t String_map.t) ~dir ~alias_module (m : Module.t) =
+let build_cm sctx ?sandbox ~dynlink ~flags ~cm_kind ~dep_graphs
+      ~requires ~dir ~alias_module (m : Module.t) =
  let ctx = SC.context sctx in
  Option.iter (Mode.of_cm_kind cm_kind |> Context.compiler ctx) ~f:(fun compiler ->
    Option.iter (Module.cm_source ~dir m cm_kind) ~f:(fun src ->
@ -36,20 +36,15 @@ let build_cm sctx ?sandbox ~dynlink ~flags ~cm_kind ~(dep_graph:Ocamldep.dep_gra
        | Cmx -> Path.relative dir (m.obj_name ^ ctx.ext_obj) :: extra_targets
        | Cmi | Cmo -> extra_targets
      in
-      let dep_graph = Ml_kind.Dict.get dep_graph ml_kind in
+      let dep_graph = Ml_kind.Dict.get dep_graphs ml_kind in
      let other_cm_files =
        Build.dyn_paths
-          (dep_graph >>^ (fun dep_graph ->
-             let deps =
-               List.map (Utils.find_deps ~dir dep_graph m.name)
-                 ~f:(Utils.find_module ~dir modules)
-             in
-             List.concat_map
-               deps
-               ~f:(fun m ->
-                 match cm_kind with
-                 | Cmi | Cmo -> [Module.cm_file m ~dir Cmi]
-                 | Cmx -> [Module.cm_file m ~dir Cmi; Module.cm_file m ~dir Cmx])))
+          (Ocamldep.Dep_graph.deps_of dep_graph m >>^ fun deps ->
+           List.concat_map deps
+             ~f:(fun m ->
+               match cm_kind with
+               | Cmi | Cmo -> [Module.cm_file m ~dir Cmi]
+               | Cmx -> [Module.cm_file m ~dir Cmi; Module.cm_file m ~dir Cmx]))
      in
      let extra_targets, cmt_args =
        match cm_kind with
@ -79,18 +74,18 @@ let build_cm sctx ?sandbox ~dynlink ~flags ~cm_kind ~(dep_graph:Ocamldep.dep_gra
           ; A "-c"; Ml_kind.flag ml_kind; Dep src
           ])))

-let build_module sctx ?sandbox ~dynlink ~js_of_ocaml ~flags m ~scope ~dir ~dep_graph
-      ~modules ~requires ~alias_module =
+let build_module sctx ?sandbox ~dynlink ~js_of_ocaml ~flags m ~scope ~dir ~dep_graphs
+      ~requires ~alias_module =
  List.iter Cm_kind.all ~f:(fun cm_kind ->
    let requires = Cm_kind.Dict.get requires cm_kind in
-    build_cm sctx ?sandbox ~dynlink ~flags ~dir ~dep_graph ~modules m ~cm_kind
+    build_cm sctx ?sandbox ~dynlink ~flags ~dir ~dep_graphs m ~cm_kind
      ~requires ~alias_module);
  (* Build *.cmo.js *)
  let src = Module.cm_file m ~dir Cm_kind.Cmo in
  SC.add_rules sctx (Js_of_ocaml_rules.build_cm sctx ~scope ~dir ~js_of_ocaml ~src)

-let build_modules sctx ~dynlink ~js_of_ocaml ~flags ~scope ~dir ~dep_graph ~modules ~requires
-      ~alias_module =
+let build_modules sctx ~dynlink ~js_of_ocaml ~flags ~scope ~dir ~dep_graphs
+      ~modules ~requires ~alias_module =
  let cmi_requires =
    Build.memoize "cmi library dependencies"
      (requires
@ -114,5 +109,5 @@ let build_modules sctx ~dynlink ~js_of_ocaml ~flags ~scope ~dir ~dep_graph ~modu
     | None -> modules
     | Some (m : Module.t) -> String_map.remove m.name modules)
    ~f:(fun ~key:_ ~data:m ->
-      build_module sctx m ~dynlink ~js_of_ocaml ~flags ~scope ~dir ~dep_graph ~modules ~requires
-        ~alias_module)
+      build_module sctx m ~dynlink ~js_of_ocaml ~flags ~scope ~dir ~dep_graphs
+        ~requires ~alias_module)
--- a/src/module_compilation.mli
+++ b/src/module_compilation.mli
@ -15,8 +15,7 @@ val build_module
  -> Module.t
  -> scope:Lib_db.Scope.t
  -> dir:Path.t
-  -> dep_graph:Ocamldep.dep_graph
-  -> modules:Module.t String_map.t
+  -> dep_graphs:Ocamldep.Dep_graphs.t
  -> requires:(unit, Lib.t list) Build.t Cm_kind.Dict.t
  -> alias_module:Module.t option
  -> unit
@ -29,7 +28,7 @@ val build_modules
  -> flags:Ocaml_flags.t
  -> scope:Lib_db.Scope.t
  -> dir:Path.t
-  -> dep_graph:Ocamldep.dep_graph
+  -> dep_graphs:Ocamldep.Dep_graphs.t
  -> modules:Module.t String_map.t
  -> requires:(unit, Lib.t list) Build.t
  -> alias_module:Module.t option
--- a/src/ocamldep.ml
+++ b/src/ocamldep.ml
@ -3,29 +3,84 @@ open Build.O

 module SC = Super_context

-type dep_graph = (unit, string list String_map.t) Build.t Ml_kind.Dict.t
+module Dep_graph = struct
+  type t =
+    { dir        : Path.t
+    ; per_module : (unit, Module.t list) Build.t String_map.t
+    }

-let parse_deps ~dir lines ~modules ~alias_module ~lib_interface_module =
-  List.map lines ~f:(fun line ->
+  let deps_of t (m : Module.t) =
+    match String_map.find m.name t.per_module with
+    | Some x -> x
+    | None ->
+      Sexp.code_error "Ocamldep.Dep_graph.deps_of"
+        [ "dir", Path.sexp_of_t t.dir
+        ; "modules", Sexp.To_sexp.(list string) (String_map.keys t.per_module)
+        ; "module", Atom m.name
+        ]
+
+  module Dep_closure =
+    Top_closure.Make(String)(struct
+      type t = Module.t
+      type graph = t list String_map.t
+      let key (t : t) = t.name
+      let deps t map = Option.value_exn (String_map.find (key t) map)
+    end)
+
+  let top_closed t modules =
+    Build.all
+      (List.map (String_map.bindings t.per_module) ~f:(fun (unit, deps) ->
+         deps >>^ fun deps -> (unit, deps)))
+    >>^ fun per_module ->
+    let per_module = String_map.of_alist_exn per_module in
+    match Dep_closure.top_closure per_module modules with
+    | Ok modules -> modules
+    | Error cycle ->
+      die "dependency cycle between modules in %s:\n   %s" (Path.to_string t.dir)
+        (String.concat ~sep:"\n-> "
+           (List.map cycle ~f:Module.name))
+
+  let dummy (m : Module.t) =
+    { dir = Path.root
+    ; per_module = String_map.singleton m.name (Build.return [])
+    }
+end
+
+module Dep_graphs = struct
+  type t = Dep_graph.t Ml_kind.Dict.t
+
+  let dummy m =
+    Ml_kind.Dict.make_both (Dep_graph.dummy m)
+end
+
+let parse_deps ~dir ~file ~(unit : Module.t)
+      ~modules ~alias_module ~lib_interface_module lines =
+  let invalid () =
+    die "ocamldep returned unexpected output for %s:\n\
+         %s"
+      (Path.to_string_maybe_quoted file)
+      (String.concat ~sep:"\n"
+         (List.map lines ~f:(sprintf "> %s")))
+  in
+  match lines with
+  | [] | _ :: _ :: _ -> invalid ()
+  | [line] ->
    match String.index line ':' with
-    | None -> die "`ocamldep` in %s returned invalid line: %S" (Path.to_string dir) line
+    | None -> invalid ()
    | Some i ->
-      let unit =
-        let basename =
-          String.sub line ~pos:0 ~len:i
-          |> Filename.basename
-        in
-        let module_basename =
-          match String.index basename '.' with
-          | None -> basename
-          | Some i -> String.sub basename ~pos:0 ~len:i
-        in
-        String.capitalize_ascii module_basename
+      let basename =
+        String.sub line ~pos:0 ~len:i
+        |> Filename.basename
      in
+      if basename <> Path.basename file then invalid ();
      let deps =
        String.extract_blank_separated_words (String.sub line ~pos:(i + 1)
                                                ~len:(String.length line - (i + 1)))
-        |> List.filter ~f:(fun m -> m <> unit && String_map.mem m modules)
+        |> List.filter_map ~f:(fun m ->
+          if m = unit.name then
+            None
+          else
+            String_map.find m modules)
      in
      (match lib_interface_module with
       | None -> ()
@ -33,75 +88,49 @@ let parse_deps ~dir lines ~modules ~alias_module ~lib_interface_module =
         let is_alias_module =
           match alias_module with
           | None -> false
-           | Some (m : Module.t) -> unit = m.name
+           | Some (m : Module.t) -> unit.name = m.name
         in
-         if unit <> m.name && not is_alias_module && List.mem m.name ~set:deps then
+         if unit.name <> m.name && not is_alias_module &&
+            List.exists deps ~f:(fun x -> Module.name x = m.name) then
           die "Module %s in directory %s depends on %s.\n\
                This doesn't make sense to me.\n\
                \n\
                %s is the main module of the library and is the only module exposed \n\
                outside of the library. Consequently, it should be the one depending \n\
                on all the other modules in the library."
-             unit (Path.to_string dir) m.name m.name);
+             unit.name (Path.to_string dir) m.name m.name);
      let deps =
        match alias_module with
        | None -> deps
-        | Some (m : Module.t) -> m.name :: deps
+        | Some m -> m :: deps
      in
-      (unit, deps))
-  |> String_map.of_alist
-  |> function
-  | Ok x -> begin
-      match alias_module with
-      | None -> x
-      | Some m -> String_map.add x ~key:m.name ~data:[]
-    end
-  | Error (unit, _, _) ->
-    die
-      "`ocamldep` in %s returned %s several times" (Path.to_string dir) unit
+      deps

-let rules sctx ~ml_kind ~dir ~item ~modules ~alias_module ~lib_interface_module =
-  let suffix = Ml_kind.suffix ml_kind in
-  let files =
-    List.filter_map (String_map.values modules) ~f:(fun m -> Module.file ~dir m ml_kind)
-    |> List.map ~f:(fun fn ->
-      match ml_kind, Filename.extension (Path.to_string fn) with
-      | Impl, ".ml"  -> Arg_spec.Dep fn
-      | Intf, ".mli" -> Dep fn
-      | Impl, _ -> S [A "-impl"; Dep fn]
-      | Intf, _ -> S [A "-intf"; Dep fn])
+let rules sctx ~ml_kind ~dir ~modules ~alias_module ~lib_interface_module =
+  let per_module =
+    String_map.map modules ~f:(fun unit ->
+      match Module.file ~dir unit ml_kind with
+      | None -> Build.return []
+      | Some file ->
+        let ocamldep_output = Path.extend_basename file ~suffix:".d" in
+        let context = SC.context sctx in
+        SC.add_rule sctx
+          (Build.run ~context (Ok context.ocamldep)
+             [A "-modules"; Ml_kind.flag ml_kind; Dep file]
+             ~stdout_to:ocamldep_output);
+        Build.memoize (Path.to_string ocamldep_output)
+          (Build.lines_of ocamldep_output
+           >>^ parse_deps ~dir ~file ~unit ~modules ~alias_module ~lib_interface_module))
  in
-  let ocamldep_output =
-    Path.relative dir (sprintf "%s.depends%s.ocamldep-output" item suffix)
+  let per_module =
+    match alias_module with
+    | None -> per_module
+    | Some m -> String_map.add per_module ~key:m.name ~data:(Build.return [])
  in
-  let ctx = SC.context sctx in
-  SC.add_rule sctx
-    (Build.run ~context:ctx (Ok ctx.ocamldep) [A "-modules"; S files]
-       ~stdout_to:ocamldep_output);
-  Build.memoize (Path.to_string ocamldep_output)
-    (Build.lines_of ocamldep_output
-     >>^ parse_deps ~dir ~modules ~alias_module ~lib_interface_module)
+  { Dep_graph.
+    dir
+  ; per_module
+  }

-module Dep_closure =
-  Top_closure.Make(String)(struct
-    type t = string
-    type graph = Path.t * t list String_map.t
-    let key t = t
-    let deps t (dir, map) = Utils.find_deps ~dir map t
-  end)
-
-let dep_closure ~dir dep_graph names =
-  match Dep_closure.top_closure (dir, dep_graph) names with
-  | Ok names -> names
-  | Error cycle ->
-    die "dependency cycle between modules in %s:\n   %s" (Path.to_string dir)
-      (String.concat cycle ~sep:"\n-> ")
-
-let names_to_top_closed_cm_files ~dir ~dep_graph ~modules ~mode names =
-  let cm_kind = Mode.cm_kind mode in
-  List.map (dep_closure ~dir dep_graph names) ~f:(fun name ->
-    let m = Utils.find_module ~dir modules name in
-    Module.cm_file m ~dir cm_kind)
-
-let rules sctx ~dir ~item ~modules ~alias_module ~lib_interface_module =
-  Ml_kind.Dict.of_func (rules sctx ~dir ~item ~modules ~alias_module ~lib_interface_module)
+let rules sctx ~dir ~modules ~alias_module ~lib_interface_module =
+  Ml_kind.Dict.of_func (rules sctx ~dir ~modules ~alias_module ~lib_interface_module)
--- a/src/ocamldep.mli
+++ b/src/ocamldep.mli
@ -2,7 +2,22 @@

 open Import

-type dep_graph = (unit, string list String_map.t) Build.t Ml_kind.Dict.t
+module Dep_graph : sig
+  type t
+
+  val deps_of
+    :  t
+    -> Module.t
+    -> (unit, Module.t list) Build.t
+
+  val top_closed : t -> Module.t list -> (unit, Module.t list) Build.t
+end
+
+module Dep_graphs : sig
+  type t = Dep_graph.t Ml_kind.Dict.t
+
+  val dummy : Module.t -> t
+end

 (** Generate ocamldep rules for the given modules. [item] is either the internal name of a
    library of the first name of a list of executables.
@ -14,17 +29,7 @@ type dep_graph = (unit, string list String_map.t) Build.t Ml_kind.Dict.t
 val rules
  :  Super_context.t
  -> dir:Path.t
-  -> item:string
  -> modules:Module.t String_map.t
  -> alias_module:Module.t option
  -> lib_interface_module:Module.t option
-  -> dep_graph
-
-(** Close and convert a list of module names to a list of .cm file names *)
-val names_to_top_closed_cm_files
-  :  dir:Path.t
-  -> dep_graph:string list String_map.t
-  -> modules:Module.t String_map.t
-  -> mode:Mode.t
-  -> string list
-  -> Path.t list
+  -> Dep_graphs.t
--- a/src/odoc.ml
+++ b/src/odoc.ml
@ -55,24 +55,27 @@ module Module_or_mld = struct
    | Module _ -> html_dir ~doc_dir t ++ "index.html"
 end

-let module_or_mld_deps (m : Module_or_mld.t) ~dir ~doc_dir ~dep_graph ~modules =
-  Build.dyn_paths
-    (dep_graph
-     >>^ fun graph ->
-     match m with
-     | Mld _ -> []
-     | Module m ->
-       List.map (Utils.find_deps ~dir graph m.name)
-         ~f:(fun name ->
-           let m = Utils.find_module ~dir modules name in
-           Module.odoc_file m ~doc_dir))
+let module_or_mld_deps (m : Module_or_mld.t) ~doc_dir
+      ~(dep_graphs:Ocamldep.Dep_graphs.t) =
+  match m with
+  | Mld _ ->
+    Build.arr (fun x -> x)
+  | Module m ->
+    Build.dyn_paths
+      ((match m.intf with
+         | Some _ ->
+           Ocamldep.Dep_graph.deps_of dep_graphs.intf m
+         | None ->
+           (* When a module has no .mli, use the dependencies for the .ml *)
+           Ocamldep.Dep_graph.deps_of dep_graphs.impl m)
+       >>^ List.map ~f:(Module.odoc_file ~doc_dir))

-let compile sctx (m : Module_or_mld.t) ~odoc ~dir ~includes ~dep_graph
-      ~doc_dir ~modules ~lib_unique_name =
+let compile sctx (m : Module_or_mld.t) ~odoc ~dir ~includes ~dep_graphs
+      ~doc_dir ~lib_unique_name =
  let context = SC.context sctx in
  let odoc_file = Module_or_mld.odoc_file m ~doc_dir in
  SC.add_rule sctx
-    (module_or_mld_deps m ~doc_dir ~dir ~dep_graph ~modules
+    (module_or_mld_deps m ~doc_dir ~dep_graphs
     >>>
     includes
     >>>
@ -155,22 +158,10 @@ let css_file ~doc_dir = doc_dir ++ "odoc.css"
 let toplevel_index ~doc_dir = doc_dir ++ "index.html"

 let setup_library_rules sctx (lib : Library.t) ~dir ~modules ~mld_files
-      ~requires ~(dep_graph:Ocamldep.dep_graph) =
+      ~requires ~(dep_graphs:Ocamldep.Dep_graph.t Ml_kind.Dict.t) =
  let doc_dir = SC.Doc.dir sctx (dir, lib) in
  let lib_unique_name = SC.unique_library_name sctx (Internal (dir, lib)) in
  let lib_name = Library.best_name lib in
-  let dep_graph =
-    Build.memoize "odoc deps"
-      ((* Use the dependency graph given by ocamldep. However, when a module has no
-          .mli, use the dependencies for the .ml *)
-        Build.fanout dep_graph.intf dep_graph.impl
-        >>^ fun (intf, impl) ->
-        String_map.merge intf impl ~f:(fun _ intf impl ->
-          match intf, impl with
-          | Some _, _    -> intf
-          | None, Some _ -> impl
-          | None, None -> assert false))
-  in
  let odoc = get_odoc sctx in
  let includes =
    Build.memoize "includes"
@ -183,12 +174,12 @@ let setup_library_rules sctx (lib : Library.t) ~dir ~modules ~mld_files
  in
  let mld_and_odoc_files =
    List.map mld_files ~f:(fun m ->
-      compile sctx ~odoc ~dir ~includes ~dep_graph ~modules
+      compile sctx ~odoc ~dir ~includes ~dep_graphs
        ~doc_dir ~lib_unique_name (Mld m))
  in
  let modules_and_odoc_files =
    List.map (String_map.values modules) ~f:(fun m ->
-      compile sctx ~odoc ~dir ~includes ~dep_graph ~modules
+      compile sctx ~odoc ~dir ~includes ~dep_graphs
        ~doc_dir ~lib_unique_name (Module m))
  in
  let inputs_and_odoc_files = modules_and_odoc_files @ mld_and_odoc_files in
--- a/src/odoc.mli
+++ b/src/odoc.mli
@ -10,7 +10,7 @@ val setup_library_rules
  -> modules:Module.t String_map.t
  -> mld_files:string list
  -> requires:(unit, Lib.t list) Build.t
-  -> dep_graph:Ocamldep.dep_graph
+  -> dep_graphs:Ocamldep.Dep_graphs.t
  -> unit

 val gen_rules : Super_context.t -> dir:Path.t -> string list -> unit
--- a/src/ordered_set_lang.ml
+++ b/src/ordered_set_lang.ml
@ -120,10 +120,17 @@ module Unexpanded = struct
    | Element s -> Element (f (String_with_vars.t s))
    | Special (l, s) -> Special (l, s)
    | Include fn ->
-      parse_general (
-        String_map.find_exn (f fn) files_contents ~string_of_key:(sprintf "%S")
-          ~desc:(fun _ -> "<filename to s-expression>")
-      ) ~f:(fun s -> f (String_with_vars.t s))
+      parse_general
+        (let fn = f fn in
+         match String_map.find fn files_contents with
+         | Some x -> x
+         | None ->
+           Sexp.code_error
+             "Ordered_set_lang.Unexpanded.expand"
+             [ "included-file", Atom fn
+             ; "files", Sexp.To_sexp.(list string) (String_map.keys files_contents)
+             ])
+        ~f:(fun s -> f (String_with_vars.t s))
    | Union l ->
      Union (List.map l ~f:(expand ~files_contents ~f))
    | Diff (l, r) ->
--- a/src/utils.ml
+++ b/src/utils.ml
@ -140,19 +140,6 @@ let g () =
  else
    []

-let find_module ~dir modules name =
-  String_map.find_exn name modules
-    ~string_of_key:(sprintf "%S")
-    ~desc:(fun _ ->
-      sprintf "<module name to module info in %s>"
-        (Path.to_string_maybe_quoted dir))
-
-let find_deps ~dir dep_graph name =
-  String_map.find_exn name dep_graph
-    ~string_of_key:(sprintf "%S")
-    ~desc:(fun _ -> sprintf "<dependency graph in %s>"
-                      (Path.to_string_maybe_quoted dir))
-
 let obj_name_of_basename fn =
  match String.index fn '.' with
  | None -> fn
--- a/src/utils.mli
+++ b/src/utils.mli
@ -39,11 +39,6 @@ val library_not_found : ?context:string -> ?hint:string -> string -> _
 (** [\["-g"\]] if [!Clflags.g] and [\[\]] otherwise *)
 val g : unit -> string list

-(** Similar to [String_map.find] but with a better error message in case of
-    failure. *)
-val find_module : dir:Path.t -> 'a String_map.t -> string -> 'a
-val find_deps   : dir:Path.t -> 'a String_map.t -> string -> 'a
-
 (** Base name of the object file (.o) for a given source file basename:

    - [obj_name_of_basename "toto.ml" = "toto"]
--- a/test/blackbox-tests/test-cases/c-stubs/run.t
+++ b/test/blackbox-tests/test-cases/c-stubs/run.t
@ -1,8 +1,8 @@
  $ $JBUILDER exec -j1 ./qnativerun/run.exe --root .
-      ocamldep qnativerun/run.depends.ocamldep-output
+      ocamldep qnativerun/run.ml.d
        ocamlc q/q_stub.o
-      ocamldep q/q.depends.ocamldep-output
-      ocamldep q/q.dependsi.ocamldep-output
+      ocamldep q/q.ml.d
+      ocamldep q/q.mli.d
    ocamlmklib q/dllq_stubs.so,q/libq_stubs.a
        ocamlc q/q.{cmi,cmti}
        ocamlc qnativerun/run.{cmi,cmo,cmt}
--- a/test/blackbox-tests/test-cases/copy_files/run.t
+++ b/test/blackbox-tests/test-cases/copy_files/run.t
@ -1,7 +1,8 @@
  $ $JBUILDER build -j1 test.exe .merlin --root . --debug-dependency-path
      ocamllex lexers/lexer1.ml
-      ocamldep test.depends.ocamldep-output
-      ocamldep foo.depends.ocamldep-output
+      ocamldep test.ml.d
+      ocamldep lexer1.ml.d
+      ocamldep dummy.ml.d
        ocamlc bar.o
        ocamlc dummy.{cmi,cmo,cmt}
    ocamlmklib dllfoo_stubs.so,libfoo_stubs.a
--- a/test/blackbox-tests/test-cases/cross-compilation/run.t
+++ b/test/blackbox-tests/test-cases/cross-compilation/run.t
@ -1,9 +1,9 @@
  $ env OCAMLFIND_CONF=$PWD/etc/findlib.conf $JBUILDER build --root . -j1 -x foo file @install
-      ocamldep bin/blah.depends.ocamldep-output [default.foo]
-      ocamldep lib/p.depends.ocamldep-output [default.foo]
-      ocamldep bin/blah.depends.ocamldep-output
+      ocamldep bin/blah.ml.d [default.foo]
+      ocamldep lib/p.ml.d [default.foo]
+      ocamldep bin/blah.ml.d
        ocamlc lib/p.{cmi,cmo,cmt} [default.foo]
-      ocamldep lib/p.depends.ocamldep-output
+      ocamldep lib/p.ml.d
      ocamlopt lib/p.{cmx,o} [default.foo]
        ocamlc bin/blah.{cmi,cmo,cmt} [default.foo]
        ocamlc lib/p.cma [default.foo]
--- a/test/blackbox-tests/test-cases/exec-cmd/run.t
+++ b/test/blackbox-tests/test-cases/exec-cmd/run.t
@ -3,7 +3,7 @@
  Error: Program "./foo.exe" isn't built yet you need to buid it first or remove the --no-build option.
  [1]
  $ $JBUILDER exec ./foo.exe -j1 --root .
-      ocamldep foo.depends.ocamldep-output
+      ocamldep foo.ml.d
        ocamlc foo.{cmi,cmo,cmt}
      ocamlopt foo.{cmx,o}
      ocamlopt foo.exe
@ -17,7 +17,7 @@
  Error: Program "bar" isn't built yet you need to buid it first or remove the --no-build option.
  [1]
  $ $JBUILDER exec bar -j1 --root .
-      ocamldep bar.depends.ocamldep-output
+      ocamldep bar.ml.d
        ocamlc bar.{cmi,cmo,cmt}
      ocamlopt bar.{cmx,o}
      ocamlopt bar.exe
--- a/test/blackbox-tests/test-cases/force-test/run.t
+++ b/test/blackbox-tests/test-cases/force-test/run.t
@ -1,6 +1,6 @@
  $ $JBUILDER clean -j1 --root .
  $ $JBUILDER runtest -j1 --root .
-      ocamldep f.depends.ocamldep-output
+      ocamldep f.ml.d
        ocamlc f.{cmi,cmo,cmt}
      ocamlopt f.{cmx,o}
      ocamlopt f.exe
--- a/test/blackbox-tests/test-cases/gen-opam-install-file/run.t
+++ b/test/blackbox-tests/test-cases/gen-opam-install-file/run.t
@ -1,7 +1,7 @@
  $ $JBUILDER runtest -j1 --root .
-      ocamldep bar.depends.ocamldep-output
-      ocamldep foo_byte.depends.ocamldep-output
-      ocamldep foo.depends.ocamldep-output
+      ocamldep bar.ml.d
+      ocamldep foo_byte.ml.d
+      ocamldep foo.ml.d
        ocamlc foo_byte.{cmi,cmo,cmt}
        ocamlc foo.{cmi,cmo,cmt}
        ocamlc foo_byte.cma
--- a/test/blackbox-tests/test-cases/github25/root/run.t
+++ b/test/blackbox-tests/test-cases/github25/root/run.t
@ -7,7 +7,6 @@ problem. So jbuilder shouldn't crash because of "plop.ca-marche-pas"
 We need ocamlfind to run this test

  $ $JBUILDER build -j1 @install --root . --only hello
-      ocamldep hello.depends.ocamldep-output
        ocamlc hello.{cmi,cmo,cmt}
      ocamlopt hello.{cmx,o}
        ocamlc hello.cma
--- a/test/blackbox-tests/test-cases/js_of_ocaml/run.t
+++ b/test/blackbox-tests/test-cases/js_of_ocaml/run.t
@ -8,8 +8,10 @@
           ppx bin/technologic.pp.ml
           ppx bin/z.pp.ml
      ocamlopt lib/x__.{cmx,o}
-      ocamldep lib/x.depends.ocamldep-output
-      ocamldep bin/technologic.depends.ocamldep-output
+      ocamldep lib/x.pp.ml.d
+      ocamldep lib/y.pp.ml.d
+      ocamldep bin/technologic.pp.ml.d
+      ocamldep bin/z.pp.ml.d
        ocamlc lib/x__Y.{cmi,cmo,cmt}
   js_of_ocaml .js/js_of_ocaml/js_of_ocaml.cma.js
   js_of_ocaml .js/stdlib/stdlib.cma.js
--- a/test/blackbox-tests/test-cases/menhir/run.t
+++ b/test/blackbox-tests/test-cases/menhir/run.t
@ -1,10 +1,15 @@
  $ $JBUILDER build -j1 src/test.exe --root . --debug-dependency-path
      ocamllex src/lexer1.ml
      ocamllex src/lexer2.ml
+      ocamldep src/test.ml.d
        menhir src/test_base.{ml,mli}
        menhir src/test_menhir1.{ml,mli}
-      ocamldep src/test.depends.ocamldep-output
-      ocamldep src/test.dependsi.ocamldep-output
+      ocamldep src/lexer1.ml.d
+      ocamldep src/lexer2.ml.d
+      ocamldep src/test_base.ml.d
+      ocamldep src/test_menhir1.ml.d
+      ocamldep src/test_menhir1.mli.d
+      ocamldep src/test_base.mli.d
        ocamlc src/test_menhir1.{cmi,cmti}
        ocamlc src/test_base.{cmi,cmti}
      ocamlopt src/test_menhir1.{cmx,o}
--- a/test/blackbox-tests/test-cases/multiple-private-libs/run.t
+++ b/test/blackbox-tests/test-cases/multiple-private-libs/run.t
@ -2,14 +2,12 @@ This test checks that there is no clash when two private libraries have the same

  $ $JBUILDER build -j1 --root . @doc
          odoc _doc/odoc.css
-      ocamldep a/test.depends.ocamldep-output
-      ocamldep a/test.dependsi.ocamldep-output
-      ocamldep b/test.depends.ocamldep-output
-      ocamldep b/test.dependsi.ocamldep-output
-        ocamlc a/test.{cmi,cmo,cmt}
          odoc _doc/test@a/page-index.odoc
-        ocamlc b/test.{cmi,cmo,cmt}
+      ocamldep a/test.ml.d
          odoc _doc/test@b/page-index.odoc
+      ocamldep b/test.ml.d
+        ocamlc a/test.{cmi,cmo,cmt}
+        ocamlc b/test.{cmi,cmo,cmt}
          odoc _doc/test@a/test.odoc
          odoc _doc/test@b/test.odoc
          odoc _doc/test@a/index.html
--- a/test/blackbox-tests/test-cases/odoc/run.t
+++ b/test/blackbox-tests/test-cases/odoc/run.t
@ -1,15 +1,13 @@
  $ $JBUILDER build @doc -j1 --root .
-      ocamldep foo_byte.depends.ocamldep-output
-      ocamldep foo_byte.dependsi.ocamldep-output
-      ocamldep foo.depends.ocamldep-output
-      ocamldep foo.dependsi.ocamldep-output
-          odoc _doc/odoc.css
-        ocamlc foo_byte.{cmi,cmo,cmt}
+      ocamldep foo_byte.ml.d
          odoc _doc/foo.byte/page-index.odoc
          odoc _doc/foo.byte/page-test.odoc
-        ocamlc foo.{cmi,cmo,cmt}
+      ocamldep foo.ml.d
          odoc _doc/foo/page-index.odoc
          odoc _doc/foo/page-test.odoc
+          odoc _doc/odoc.css
+        ocamlc foo_byte.{cmi,cmo,cmt}
+        ocamlc foo.{cmi,cmo,cmt}
          odoc _doc/foo.byte/foo_byte.odoc
          odoc _doc/foo/foo.odoc
          odoc _doc/foo.byte/index.html
--- a/test/blackbox-tests/test-cases/ppx-rewriter/run.t
+++ b/test/blackbox-tests/test-cases/ppx-rewriter/run.t
@ -1,18 +1,18 @@
  $ $JBUILDER build ./w_omp_driver.exe -j1 --root .
-      ocamldep ppx/fooppx.depends.ocamldep-output
+      ocamldep ppx/fooppx.ml.d
        ocamlc ppx/fooppx.{cmi,cmo,cmt}
      ocamlopt ppx/fooppx.{cmx,o}
      ocamlopt ppx/fooppx.{a,cmxa}
      ocamlopt .ppx/fooppx@/ppx.exe
           ppx w_omp_driver.pp.ml
-      ocamldep w_omp_driver.depends.ocamldep-output
+      ocamldep w_omp_driver.pp.ml.d
        ocamlc w_omp_driver.{cmi,cmo,cmt}
      ocamlopt w_omp_driver.{cmx,o}
      ocamlopt w_omp_driver.exe
  $ $JBUILDER build ./w_ppx_driver.exe -j1 --root .
      ocamlopt .ppx/ppx_driver.runner/ppx.exe
           ppx w_ppx_driver.pp.ml
-      ocamldep w_ppx_driver.depends.ocamldep-output
+      ocamldep w_ppx_driver.pp.ml.d
        ocamlc w_ppx_driver.{cmi,cmo,cmt}
      ocamlopt w_ppx_driver.{cmx,o}
      ocamlopt w_ppx_driver.exe
--- a/test/blackbox-tests/test-cases/scope-bug/run.t
+++ b/test/blackbox-tests/test-cases/scope-bug/run.t
@ -1,8 +1,9 @@
  $ $JBUILDER build -j1 --root . @install
-      ocamldep alib/alib.depends.ocamldep-output
+      ocamldep alib/alib.ml.d
+      ocamldep alib/main.ml.d
        ocamlc alib/alib__.{cmi,cmo,cmt}
-      ocamldep blib/blib.depends.ocamldep-output
-      ocamldep blib/sub/sub.depends.ocamldep-output
+      ocamldep blib/blib.ml.d
+      ocamldep blib/sub/sub.ml.d
      ocamlopt alib/alib__.{cmx,o}
        ocamlc blib/sub/sub.{cmi,cmo,cmt}
      ocamlopt blib/sub/sub.{cmx,o}
--- a/test/blackbox-tests/test-cases/scope-ppx-bug/run.t
+++ b/test/blackbox-tests/test-cases/scope-ppx-bug/run.t
@ -1,7 +1,5 @@
  $ $JBUILDER build -j1 --root . @install
-      ocamldep a/ppx/a.depends.ocamldep-output
        ocamlc a/ppx/a.{cmi,cmo,cmt}
-      ocamldep a/kernel/a_kernel.depends.ocamldep-output
        ocamlc a/kernel/a_kernel.{cmi,cmo,cmt}
      ocamlopt a/ppx/a.{cmx,o}
        ocamlc a/ppx/a.cma
@ -14,7 +12,7 @@
      ocamlopt .ppx/a.kernel/ppx.exe
      ocamlopt .ppx/a/ppx.exe
           ppx b/b.pp.ml
-      ocamldep b/b.depends.ocamldep-output
+      ocamldep b/b.pp.ml.d
        ocamlc b/b.{cmi,cmo,cmt}
      ocamlopt b/b.{cmx,o}
        ocamlc b/b.cma
--- a/test/blackbox-tests/test-cases/select/run.t
+++ b/test/blackbox-tests/test-cases/select/run.t
@ -1,5 +1,11 @@
  $ $JBUILDER runtest -j1 --root .
-      ocamldep main.depends.ocamldep-output
+      ocamldep bar.ml.d
+      ocamldep bar_no_unix.ml.d
+      ocamldep bar_unix.ml.d
+      ocamldep foo.ml.d
+      ocamldep foo_fake.ml.d
+      ocamldep foo_no_fake.ml.d
+      ocamldep main.ml.d
        ocamlc bar.{cmi,cmo,cmt}
        ocamlc foo.{cmi,cmo,cmt}
      ocamlopt bar.{cmx,o}
--- a/test/blackbox-tests/test-cases/utop/run.t
+++ b/test/blackbox-tests/test-cases/utop/run.t
@ -1,6 +1,6 @@
  $ $JBUILDER utop -j1 --root . forutop -- init_forutop.ml
-      ocamldep forutop/.utop/utop.depends.ocamldep-output
-      ocamldep forutop/forutop.depends.ocamldep-output
+      ocamldep forutop/.utop/utop.ml.d
+      ocamldep forutop/forutop.ml.d
        ocamlc forutop/forutop.{cmi,cmo,cmt}
        ocamlc forutop/.utop/utop.{cmi,cmo,cmt}
        ocamlc forutop/forutop.cma