Merge pull request #1035 from diml/move-pp-files-back

Move back .pp files to the source directory

CHANGES.md

@@ -5,9 +5,6 @@ next
   passing in `--root` in conjunction with `--workspace` or `--config` would not
   work correctly (#997, @rgrinberg)
 
-- Change the location of preprocessed files inside the build directory
-  (#1004, @diml)
-
 - Fix parsing of `%{lib:name:file}` forms (#1022, fixes #1019, @diml)
 
 1.0.0 (10/07/2018)

src/gen_rules.ml

@@ -164,7 +164,7 @@ module Gen(P : Install_rules.Params) = struct
     (* Preprocess before adding the alias module as it doesn't need
        preprocessing *)
     let pp =
-      Preprocessing.make sctx ~dir ~obj_dir ~dep_kind ~scope
+      Preprocessing.make sctx ~dir ~dep_kind ~scope
         ~preprocess:lib.buildable.preprocess
         ~preprocessor_deps:
           (SC.Deps.interpret sctx ~scope ~dir
@@ -437,7 +437,7 @@ module Gen(P : Install_rules.Params) = struct
         ~scope ~dir
     in
     let pp =
-      Preprocessing.make sctx ~dir ~obj_dir ~dep_kind:Required
+      Preprocessing.make sctx ~dir ~dep_kind:Required
         ~scope
         ~preprocess:exes.buildable.preprocess
         ~preprocessor_deps

src/preprocessing.ml

@@ -4,15 +4,15 @@ open Jbuild
 
 module SC = Super_context
 
-let pped_path path ~obj_dir ~suffix =
+let pped_path path ~suffix =
   (* We need to insert the suffix before the extension as some tools
      inspect the extension *)
-  let base, ext = Filename.split_extension (Path.basename path) in
+  let base, ext = Path.split_extension path in
-  Path.relative obj_dir (base ^ suffix ^ ext)
+  Path.extend_basename base ~suffix:(suffix ^ ext)
 
-let pped_module m ~obj_dir ~f =
+let pped_module m ~f =
   Module.map_files m ~f:(fun kind file ->
-    let pp_path = pped_path file.path ~obj_dir ~suffix:".pp" in
+    let pp_path = pped_path file.path ~suffix:".pp" in
     f kind file.path pp_path;
     { file with path = pp_path })
 
@@ -407,7 +407,7 @@ let cookie_library_name lib_name =
 
 (* Generate rules for the reason modules in [modules] and return a
    a new module with only OCaml sources *)
-let setup_reason_rules sctx ~obj_dir (m : Module.t) =
+let setup_reason_rules sctx (m : Module.t) =
   let ctx = SC.context sctx in
   let refmt =
     Artifacts.binary (SC.artifacts sctx) "refmt" ~hint:"opam install reason" in
@@ -423,12 +423,22 @@ let setup_reason_rules sctx ~obj_dir (m : Module.t) =
     match f.syntax with
     | OCaml  -> f
     | Reason ->
-      let ml =
+      let path =
-        { Module.File.
+        let base, ext = Path.split_extension f.path in
-          syntax = OCaml
+        let suffix =
-        ; path   = pped_path f.path ~obj_dir ~suffix:".ast"
+          match ext with
-        }
+          | ".re"  -> ".re.ml"
+          | ".rei" -> ".re.mli"
+          | _     ->
+            Loc.fail
+              (Loc.in_file
+                 (Path.to_string (Path.drop_build_context_exn f.path)))
+              "Unknown file extension for reason source file: %S"
+              ext
+        in
+        Path.extend_basename base ~suffix
       in
+      let ml = Module.File.make OCaml path in
       SC.add_rule sctx (rule f.path ml.path);
       ml)
 
@@ -517,7 +527,7 @@ type t = (Module.t -> lint:bool -> Module.t) Per_module.t
 
 let dummy = Per_module.for_all (fun m ~lint:_ -> m)
 
-let make sctx ~dir ~obj_dir ~dep_kind ~lint ~preprocess
+let make sctx ~dir ~dep_kind ~lint ~preprocess
       ~preprocessor_deps ~lib_name ~scope ~dir_kind =
   let preprocessor_deps =
     Build.memoize "preprocessor deps" preprocessor_deps
@@ -529,13 +539,13 @@ let make sctx ~dir ~obj_dir ~dep_kind ~lint ~preprocess
   Per_module.map preprocess ~f:(function
     | Preprocess.No_preprocessing ->
       (fun m ~lint ->
-         let ast = setup_reason_rules sctx m ~obj_dir in
+         let ast = setup_reason_rules sctx m in
          if lint then lint_module ~ast ~source:m;
          ast)
     | Action (loc, action) ->
       (fun m ~lint ->
          let ast =
-           pped_module m ~obj_dir ~f:(fun _kind src dst ->
+           pped_module m ~f:(fun _kind src dst ->
              let bindings = Pform.Map.input_file src in
              SC.add_rule sctx
                (preprocessor_deps
@@ -554,9 +564,9 @@ let make sctx ~dir ~obj_dir ~dep_kind ~lint ~preprocess
                   ~dep_kind
                   ~bindings
                   ~targets:(Static [dst])
-                  ~targets_dir:obj_dir
+                  ~targets_dir:dir
                   ~scope))
-           |> setup_reason_rules sctx ~obj_dir in
+           |> setup_reason_rules sctx in
          if lint then lint_module ~ast ~source:m;
          ast)
     | Pps { loc; pps; flags } ->
@@ -582,9 +592,9 @@ let make sctx ~dir ~obj_dir ~dep_kind ~lint ~preprocess
               ~standard:(Build.return [])))
       in
       (fun m ~lint ->
-         let ast = setup_reason_rules sctx m ~obj_dir in
+         let ast = setup_reason_rules sctx m in
          if lint then lint_module ~ast ~source:m;
-         pped_module ast ~obj_dir ~f:(fun kind src dst ->
+         pped_module ast ~f:(fun kind src dst ->
            SC.add_rule sctx
              (promote_correction ~suffix:corrected_suffix
                 (Option.value_exn (Module.file m kind))

src/preprocessing.mli

@@ -10,7 +10,6 @@ val dummy : t
 val make
   :  Super_context.t
   -> dir:Path.t
-  -> obj_dir:Path.t
   -> dep_kind:Build.lib_dep_kind
   -> lint:Jbuild.Preprocess_map.t
   -> preprocess:Jbuild.Preprocess_map.t

src/stdune/path.ml

@@ -47,6 +47,8 @@ module External : sig
   val initial_cwd : t
   val cwd : unit -> t
   val extend_basename : t -> suffix:string -> t
+  val extension : t -> string
+  val split_extension : t -> t * string
 end = struct
   include Interned.No_interning(struct
       let initial_size = 512
@@ -116,6 +118,11 @@ end = struct
   let basename t = Filename.basename (to_string t)
   let parent t = as_string ~f:Filename.dirname t
 
+  let extension t = Filename.extension (to_string t)
+  let split_extension t =
+    let s, ext = Filename.split_extension (to_string t) in
+    (make s, ext)
+
   let cwd () = make (Sys.getcwd ())
   let initial_cwd = cwd ()
 end
@@ -140,6 +147,8 @@ module Local : sig
   val reach : t -> from:t -> string
   val basename : t -> string
   val extend_basename : t -> suffix:string -> t
+  val extension : t -> string
+  val split_extension : t -> t * string
   module Set : Set.S with type elt = t
 
   module Prefix : sig
@@ -344,6 +353,11 @@ end = struct
 
   let extend_basename t ~suffix = make (to_string t ^ suffix)
 
+  let extension t = Filename.extension (to_string t)
+  let split_extension t =
+    let s, ext = Filename.split_extension (to_string t) in
+    (make s, ext)
+
   module Prefix = struct
     let make_path = make
 
@@ -901,7 +915,22 @@ let compare x y =
   | _               , In_source_tree _ -> Gt
   | In_build_dir x  , In_build_dir y   -> Local.compare_val x y
 
-let extension t = Filename.extension (to_string t)
+let extension t =
+  match t with
+  | External t -> External.extension t
+  | In_build_dir t | In_source_tree t -> Local.extension t
+
+let split_extension t =
+  match t with
+  | External t ->
+    let t, ext = External.split_extension t in
+    (external_ t, ext)
+  | In_build_dir t ->
+    let t, ext = Local.split_extension t in
+    (in_build_dir t, ext)
+  | In_source_tree t ->
+    let t, ext = Local.split_extension t in
+    (in_source_tree t, ext)
 
 let pp ppf t = Format.pp_print_string ppf (to_string_maybe_quoted t)
 

src/stdune/path.mli

@@ -140,6 +140,7 @@ val rm_rf : t -> unit
 val mkdir_p : t -> unit
 
 val extension : t -> string
+val split_extension : t -> t * string
 
 val pp : Format.formatter -> t -> unit
 val pp_debug : Format.formatter -> t -> unit

test/blackbox-tests/test-cases/dune-ppx-driver-system/run.t

@@ -41,7 +41,7 @@ Same, but with error pointing to .ppx
 Test the argument syntax
 
   $ dune build test_ppx_args.cma
-           ppx .test_ppx_args.objs/test_ppx_args.pp.ml
+           ppx test_ppx_args.pp.ml
   .ppx/driver_print_args@foo/ppx.exe
   -arg1
   -arg2
@@ -50,9 +50,9 @@ Test the argument syntax
   --cookie
   library-name="test_ppx_args"
   -o
-  .test_ppx_args.objs/test_ppx_args.pp.ml
+  test_ppx_args.pp.ml
   --impl
   test_ppx_args.ml
   Error: Rule failed to generate the following targets:
-  - .test_ppx_args.objs/test_ppx_args.pp.ml
+  - test_ppx_args.pp.ml
   [1]

test/blackbox-tests/test-cases/js_of_ocaml/run.t

@@ -2,17 +2,17 @@
         ocamlc lib/stubs$ext_obj
     ocamlmklib lib/dllx_stubs$ext_dll,lib/libx_stubs$ext_lib
       ocamlopt .ppx/js_of_ocaml-ppx/ppx.exe
-           ppx lib/.x.objs/x.pp.ml
+           ppx lib/x.pp.ml
       ocamldep lib/.x.objs/x.pp.ml.d
         ocamlc lib/.x.objs/x__.{cmi,cmo,cmt}
       ocamlopt lib/.x.objs/x__.{cmx,o}
-           ppx lib/.x.objs/y.pp.ml
+           ppx lib/y.pp.ml
       ocamldep lib/.x.objs/y.pp.ml.d
         ocamlc lib/.x.objs/x__Y.{cmi,cmo,cmt}
       ocamlopt lib/.x.objs/x__Y.{cmx,o}
-           ppx bin/.technologic.eobjs/technologic.pp.ml
+           ppx bin/technologic.pp.ml
       ocamldep bin/.technologic.eobjs/technologic.pp.ml.d
-           ppx bin/.technologic.eobjs/z.pp.ml
+           ppx bin/z.pp.ml
       ocamldep bin/.technologic.eobjs/z.pp.ml.d
    js_of_ocaml .js/js_of_ocaml/js_of_ocaml.cma.js
    js_of_ocaml lib/.x.objs/x__Y.cmo.js

test/blackbox-tests/test-cases/ppx-rewriter/run.t

@@ -4,7 +4,7 @@
       ocamlopt ppx/.fooppx.objs/fooppx.{cmx,o}
       ocamlopt ppx/fooppx.{a,cmxa}
       ocamlopt .ppx/jbuild/fooppx/ppx.exe
-           ppx .w_omp_driver.eobjs/w_omp_driver.pp.ml
+           ppx w_omp_driver.pp.ml
       ocamldep .w_omp_driver.eobjs/w_omp_driver.pp.ml.d
         ocamlc .w_omp_driver.eobjs/w_omp_driver.{cmi,cmo,cmt}
       ocamlopt .w_omp_driver.eobjs/w_omp_driver.{cmx,o}

test/blackbox-tests/test-cases/private-public-overlap/run.t

@@ -15,7 +15,7 @@ On the other hand, public libraries may have private preprocessors
       ocamlopt .ppx_internal.objs/ppx_internal.{cmx,o}
       ocamlopt ppx_internal.{a,cmxa}
       ocamlopt .ppx/jbuild/ppx_internal@mylib/ppx.exe
-           ppx .mylib.objs/mylib.pp.ml
+           ppx mylib.pp.ml
       ocamldep .mylib.objs/mylib.pp.ml.d
         ocamlc .mylib.objs/mylib.{cmi,cmo,cmt}
       ocamlopt .mylib.objs/mylib.{cmx,o}
@@ -33,7 +33,7 @@ Unless they introduce private runtime dependencies:
       ocamlopt .private_ppx.objs/private_ppx.{cmx,o}
       ocamlopt private_ppx.{a,cmxa}
       ocamlopt .ppx/jbuild/private_ppx@mylib/ppx.exe
-           ppx .mylib.objs/mylib.pp.ml
+           ppx mylib.pp.ml
       ocamldep .mylib.objs/mylib.pp.ml.d
   [1]
 

test/blackbox-tests/test-cases/reason/run.t

@@ -1,54 +1,54 @@
   $ dune build @runtest @install-file --display short
-         refmt .rlib.objs/bar.ast.re
+         refmt bar.re.ml
-      ocamldep .rlib.objs/bar.ast.re.d
+      ocamldep .rlib.objs/bar.re.ml.d
       ocamldep pp/.reasononlypp.eobjs/reasononlypp.ml.d
         ocamlc pp/.reasononlypp.eobjs/reasononlypp.{cmi,cmo,cmt}
       ocamlopt pp/.reasononlypp.eobjs/reasononlypp.{cmx,o}
       ocamlopt pp/reasononlypp.exe
-  reasononlypp .rlib.objs/cppome.pp.re
+  reasononlypp cppome.pp.re
-         refmt .rlib.objs/cppome.pp.ast.re
+         refmt cppome.pp.re.ml
-      ocamldep .rlib.objs/cppome.pp.ast.re.d
+      ocamldep .rlib.objs/cppome.pp.re.ml.d
       ocamldep ppx/.reasonppx.objs/reasonppx.ml.d
         ocamlc ppx/.reasonppx.objs/reasonppx.{cmi,cmo,cmt}
       ocamlopt ppx/.reasonppx.objs/reasonppx.{cmx,o}
       ocamlopt ppx/reasonppx.{a,cmxa}
       ocamlopt .ppx/jbuild/reasonppx@rlib/ppx.exe
-           ppx .rlib.objs/foo.pp.ml
+           ppx foo.pp.ml
       ocamldep .rlib.objs/foo.pp.ml.d
-         refmt .rlib.objs/hello.ast.re
+         refmt hello.re.ml
-           ppx .rlib.objs/hello.ast.pp.re
+           ppx hello.re.pp.ml
-      ocamldep .rlib.objs/hello.ast.pp.re.d
+      ocamldep .rlib.objs/hello.re.pp.ml.d
-         refmt .rlib.objs/pped.ast.re
+         refmt pped.re.ml
-      ocamldep .rlib.objs/pped.ast.re.d
+      ocamldep .rlib.objs/pped.re.ml.d
         ocamlc .rlib.objs/rlib.{cmi,cmo,cmt}
       ocamlopt .rlib.objs/rlib.{cmx,o}
       ocamldep .rlib.objs/bar.mli.d
         ocamlc .rlib.objs/rlib__Bar.{cmi,cmti}
       ocamlopt .rlib.objs/rlib__Bar.{cmx,o}
-         refmt .rlib.objs/foo.ast.rei
+         refmt foo.re.mli
-           ppx .rlib.objs/foo.ast.pp.rei
+           ppx foo.re.pp.mli
-      ocamldep .rlib.objs/foo.ast.pp.rei.d
+      ocamldep .rlib.objs/foo.re.pp.mli.d
         ocamlc .rlib.objs/rlib__Foo.{cmi,cmti}
       ocamlopt .rlib.objs/rlib__Foo.{cmx,o}
-         refmt .rlib.objs/hello.ast.rei
+         refmt hello.re.mli
-           ppx .rlib.objs/hello.ast.pp.rei
+           ppx hello.re.pp.mli
-      ocamldep .rlib.objs/hello.ast.pp.rei.d
+      ocamldep .rlib.objs/hello.re.pp.mli.d
         ocamlc .rlib.objs/rlib__Hello.{cmi,cmti}
       ocamlopt .rlib.objs/rlib__Hello.{cmx,o}
-         refmt .rlib.objs/pped.ast.rei
+         refmt pped.re.mli
-      ocamldep .rlib.objs/pped.ast.rei.d
+      ocamldep .rlib.objs/pped.re.mli.d
         ocamlc .rlib.objs/rlib__Pped.{cmi,cmti}
       ocamlopt .rlib.objs/rlib__Pped.{cmx,o}
-  reasononlypp .rlib.objs/cppome.pp.rei
+  reasononlypp cppome.pp.rei
-         refmt .rlib.objs/cppome.pp.ast.rei
+         refmt cppome.pp.re.mli
-      ocamldep .rlib.objs/cppome.pp.ast.rei.d
+      ocamldep .rlib.objs/cppome.pp.re.mli.d
         ocamlc .rlib.objs/rlib__Cppome.{cmi,cmti}
       ocamlopt .rlib.objs/rlib__Cppome.{cmx,o}
       ocamlopt rlib.{a,cmxa}
       ocamlopt rlib.cmxs
-  reasononlypp .rbin.eobjs/rbin.pp.re
+  reasononlypp rbin.pp.re
-         refmt .rbin.eobjs/rbin.pp.ast.re
+         refmt rbin.pp.re.ml
-      ocamldep .rbin.eobjs/rbin.pp.ast.re.d
+      ocamldep .rbin.eobjs/rbin.pp.re.ml.d
         ocamlc .rbin.eobjs/rbin.{cmi,cmo,cmt}
       ocamlopt .rbin.eobjs/rbin.{cmx,o}
       ocamlopt rbin.exe

test/blackbox-tests/test-cases/scope-ppx-bug/run.t

@@ -11,7 +11,7 @@
         ocamlc a/kernel/a_kernel.cma
       ocamlopt .ppx/jbuild/a.kernel/ppx.exe
       ocamlopt .ppx/jbuild/a/ppx.exe
-           ppx b/.b.objs/b.pp.ml
+           ppx b/b.pp.ml
       ocamldep b/.b.objs/b.pp.ml.d
         ocamlc b/.b.objs/b.{cmi,cmo,cmt}
       ocamlopt b/.b.objs/b.{cmx,o}