Simplify overrides in the bootstrap path

- vendored libraries are replaced by files in `vendor/boot`
- files named `XXX.boot.EXT` replace their `XXX.EXT` equivalent

For instance, to build boot.exe, we use `src/glob_lexer.boot.ml`
rather than `src/glob_lexer.ml` (which would be generated from
`src/glob_lexer.mll`).

README.md

@@ -253,6 +253,13 @@ In order to build itself, Jbuilder uses an OCaml script
 single `boot.ml` file. This file is built using `ocamlopt` or `ocamlc`
 and used to build everything else.
 
+Note that we don't include all of the sources in boot.ml. We skip a
+few parts to speed up the build. In particular:
+- vendored libraries are replaced by simpler implementations taken
+  from `vendor/boot`
+- a few files in `src` have an alternative version. These alternatives
+  versions are named `XXX.boot.EXT`. For instance: `glob_lexer.boot.ml`
+
 ### OCaml compatibility test
 
 Install opam switches for all the entries in the

bootstrap.ml

@@ -11,23 +11,27 @@ module String = struct
     let capitalize_ascii   = String.capitalize
     let uncapitalize_ascii = String.uncapitalize
   end
+
+  let break s ~pos =
+    (sub s ~pos:0 ~len:pos,
+     sub s ~pos ~len:(String.length s - pos))
 end
 
+let dirs =
+  [ "vendor/boot"
+  ; "src"
+  ]
+
 open Printf
 
 module String_set = Set.Make(String)
+module String_map = Map.Make(String)
 
 let () =
   match Sys.getenv "OCAMLPARAM" with
   | s -> Printf.eprintf "OCAMLPARAM is set to %S\n%!" s
   | exception Not_found -> ()
 
-(* Modules overriden to bootstrap faster *)
-let overridden =
-  String_set.of_list
-    [ "Glob_lexer"
-    ]
-
 let ( ^/ ) = Filename.concat
 
 let protectx x ~finally ~f =
@@ -113,35 +117,69 @@ let bin_dir, mode, compiler =
 let ocamllex = get_prog bin_dir "ocamllex"
 let ocamldep = get_prog bin_dir "ocamldep"
 
-let run_ocamllex name =
-  let src = "src" ^/ name ^ ".mll" in
-  let dst = "src" ^/ name ^ ".ml"  in
+let run_ocamllex src =
+  let dst = String.sub src ~pos:0 ~len:(String.length src - 1) in
   let x = Sys.file_exists dst in
   let n = exec "%s -q %s" ocamllex src in
   if n <> 0 then exit n;
   if not x then
-    at_exit (fun () -> try Sys.remove dst with _ -> ())
+    at_exit (fun () -> try Sys.remove dst with _ -> ());
+  dst
 
+type module_files =
+  { impl : string
+  ; intf : string option
+  }
+
+(* Map from module names to ml/mli filenames *)
 let modules =
-  Sys.readdir "src"
-  |> Array.fold_left ~init:[] ~f:(fun acc fn ->
-    match String.rindex fn '.' with
-    | exception Not_found -> acc
-    | i ->
-      let ext = String.sub fn ~pos:(i + 1) ~len:(String.length fn - i - 1) in
-      match ext with
-      | "ml" | "mll" ->
-        let base = String.sub fn ~pos:0 ~len:i in
-        let mod_name = String.capitalize_ascii base in
-        if String_set.mem mod_name overridden then
-          acc
-        else begin
-          if ext = "mll" then run_ocamllex base;
-          String.capitalize_ascii base :: acc
-        end
-      | _ ->
-        acc)
-  |> String_set.of_list
+  let files_of dir =
+    Sys.readdir dir |> Array.to_list |> List.map ~f:(Filename.concat dir)
+  in
+  let impls, intfs =
+    List.map dirs ~f:files_of
+    |> List.concat
+    |> List.fold_left ~init:(String_map.empty, String_map.empty)
+         ~f:(fun ((impls, intfs) as acc) fn ->
+           let base = Filename.basename fn in
+           match String.index base '.' with
+           | exception Not_found -> acc
+           | i ->
+             let base, ext = String.break base i in
+             let is_boot, ext =
+               match String.rindex ext '.' with
+               | exception Not_found -> (false, ext)
+               | i ->
+                 let a, b = String.break ext i in
+                 if a = ".boot" then
+                   (true, b)
+                 else
+                   (false, ext)
+             in
+             match ext with
+             | ".ml" | ".mll" ->
+               let mod_name = String.capitalize_ascii base in
+               if is_boot || not (String_map.mem mod_name impls) then
+                 let fn =
+                   if ext = ".mll" then lazy (run_ocamllex fn) else lazy fn
+                 in
+                 (String_map.add mod_name fn impls, intfs)
+               else
+                 acc
+             | ".mli" ->
+               let mod_name = String.capitalize_ascii base in
+               if is_boot || not (String_map.mem mod_name intfs) then
+                 (impls, String_map.add mod_name fn intfs)
+               else
+                 acc
+             | _ -> acc)
+  in
+  String_map.merge
+    (fun _ impl intf ->
+       match impl with
+       | None -> None
+       | Some impl -> Some { impl = Lazy.force impl; intf })
+    impls intfs
 
 let split_words s =
   let rec skip_blanks i =
@@ -187,13 +225,13 @@ let read_deps files =
     let unit =
       String.sub line ~pos:0 ~len:i
       |> Filename.basename
-      |> Filename.chop_extension
+      |> (fun s -> String.sub s ~pos:0 ~len:(String.index s '.'))
       |> String.capitalize_ascii
     in
     let deps =
       split_words (String.sub line ~pos:(i + 1)
                      ~len:(String.length line - (i + 1)))
-      |> List.filter ~f:(fun m -> String_set.mem m modules)
+      |> List.filter ~f:(fun m -> String_map.mem m modules)
     in
     (unit, deps))
 
@@ -216,10 +254,9 @@ let topsort deps =
   done;
   List.rev !res
 
-let modules =
+let topsorted_module_names =
   let files =
-    List.map (String_set.elements modules) ~f:(fun unit ->
-      sprintf "src/%s.ml" (String.uncapitalize_ascii unit))
+    List.map (String_map.bindings modules) ~f:(fun (_, x) -> x.impl)
   in
   topsort (read_deps files)
 
@@ -263,68 +300,24 @@ let generate_file_with_all_the_sources () =
     pos_in_generated_file := !pos_in_generated_file + newlines;
     pr "# %d %S" (!pos_in_generated_file + 1) generated_file
   in
-  let s = {|
-module Jbuilder_re = struct
-  module Re = struct
-    type t = unit
-    type re = unit
-    let compile () = ()
-    let execp _ _ = false
-  end
-end
-
-module Jbuilder_opam_file_format = struct
-  module OpamParserTypes = struct
-    type value =
-      | String of unit * string
-      | List of unit * value list
-      | Other
-
-    type opamfile_item =
-      | Variable of unit * string * value
-      | Other
-
-    type opamfile =
-      { file_contents : opamfile_item list
-      ; file_name     : string
-      }
-  end
-  module OpamBaseParser = struct
-    open OpamParserTypes
-    let main _lex _lexbuf fn =
-      assert (fn = "jbuilder.opam");
-      { file_contents = []
-      ; file_name     = fn
-      }
-  end
-  module OpamLexer = struct
-    exception Error of string
-    let token _ = assert false
-  end
-end
-
-module Glob_lexer = struct
-  let parse_string _ = failwith "globs are not available during bootstrap"
-end
-|}
-  in
-  output_string oc s;
-  pos_in_generated_file := !pos_in_generated_file + count_newlines s;
-  List.iter modules ~f:(fun m ->
-    let base = String.uncapitalize_ascii m in
-    let mli = sprintf "src/%s.mli" base in
-    let ml  = sprintf "src/%s.ml"  base in
-    if Sys.file_exists mli then begin
+  List.iter topsorted_module_names ~f:(fun m ->
+    let { impl; intf } =
+      try
+        String_map.find m modules
+      with Not_found ->
+        Printf.ksprintf failwith "module not found: %s" m
+    in
+    match intf with
+    | Some intf ->
       pr "module %s : sig" m;
-      dump mli;
+      dump intf;
       pr "end = struct";
-      dump ml;
+      dump impl;
       pr "end"
-    end else begin
+    | None ->
       pr "module %s = struct" m;
-      dump ml;
-      pr "end"
-    end);
+      dump impl;
+      pr "end");
   output_string oc "let () = Main.bootstrap ()\n";
   close_out oc
 

src/glob_lexer.boot.ml

@@ -0,0 +1,4 @@
+(* We don't use globs in the bootstrap path to avoid avoid to include
+   ocaml-re. This speeds up the bootstrap. *)
+
+let parse_string _ = failwith "globs are not available during bootstrap"

vendor/boot/jbuilder_opam_file_format.ml

@@ -0,0 +1,27 @@
+module OpamParserTypes = struct
+  type value =
+    | String of unit * string
+    | List of unit * value list
+    | Other
+
+  type opamfile_item =
+    | Variable of unit * string * value
+    | Other
+
+  type opamfile =
+    { file_contents : opamfile_item list
+    ; file_name     : string
+    }
+end
+module OpamBaseParser = struct
+  open OpamParserTypes
+  let main _lex _lexbuf fn =
+    assert (fn = "jbuilder.opam");
+    { file_contents = []
+    ; file_name     = fn
+    }
+end
+module OpamLexer = struct
+  exception Error of string
+  let token _ = assert false
+end

vendor/boot/jbuilder_re.ml

@@ -0,0 +1,6 @@
+module Re = struct
+  type t = unit
+  type re = unit
+  let compile () = ()
+  let execp _ _ = false
+end