Add a --no-buffer option

Following #107

bin/main.ml

@@ -9,17 +9,18 @@ let () = suggest_function := Jbuilder_cmdliner.Cmdliner_suggest.value
 let (>>=) = Future.(>>=)
 
 type common =
-  { concurrency    : int
+  { concurrency     : int
-  ; debug_dep_path : bool
+  ; debug_dep_path  : bool
-  ; debug_findlib  : bool
+  ; debug_findlib   : bool
-  ; dev_mode       : bool
+  ; dev_mode        : bool
-  ; verbose        : bool
+  ; verbose         : bool
-  ; workspace_file : string option
+  ; workspace_file  : string option
-  ; root           : string
+  ; root            : string
-  ; target_prefix  : string
+  ; target_prefix   : string
-  ; only_packages  : String_set.t option
+  ; only_packages   : String_set.t option
+  ; capture_outputs : bool
   ; (* Original arguments for the external-lib-deps hint *)
-    orig_args      : string list
+    orig_args       : string list
   }
 
 let prefix_target common s = common.target_prefix ^ s
@@ -30,6 +31,7 @@ let set_common c ~targets =
   Clflags.debug_findlib := c.debug_findlib;
   Clflags.dev_mode := c.dev_mode;
   Clflags.verbose := c.verbose;
+  Clflags.capture_outputs := c.capture_outputs;
   Clflags.workspace_root := c.root;
   if c.root <> Filename.current_dir_name then
     Sys.chdir c.root;
@@ -109,6 +111,7 @@ let common =
         debug_findlib
         dev_mode
         verbose
+        no_buffer
         workspace_file
         (root, only_packages, orig)
     =
@@ -129,6 +132,7 @@ let common =
     ; debug_findlib
     ; dev_mode
     ; verbose
+    ; capture_outputs = not no_buffer
     ; workspace_file
     ; root
     ; orig_args
@@ -184,6 +188,23 @@ let common =
          & info ["verbose"] ~docs
              ~doc:"Print detailed information about commands being run")
   in
+  let no_buffer =
+    Arg.(value
+         & flag
+         & info ["no-buffer"] ~docs ~docv:"DIR"
+             ~doc:{|Do not buffer the output of commands executed by jbuilder.
+                    By default jbuilder buffers the output of subcommands, in order
+                    to prevent interleaving when multiple commands are executed
+                    in parallel. However, this can be an issue when debugging
+                    long running tests. With $(b,--no-buffer), commands have direct
+                    access to the terminal. Note that as a result their output won't
+                    be captured in the log file.
+
+                    You should use this option in conjunction with $(b,-j 1),
+                    to avoid interleaving. Additionally you should use
+                    $(b,--verbose) as well, to make sure that commands are printed
+                    before they are being executed.|})
+  in
   let workspace_file =
     Arg.(value
          & opt (some file) None
@@ -242,6 +263,7 @@ let common =
         $ dfindlib
         $ dev
         $ verbose
+        $ no_buffer
         $ workspace_file
         $ root_and_only_packages
        )

src/ansi_color.ml

@@ -154,7 +154,7 @@ let strip_colors_for_stderr s =
    colors. Since we support colors in the output of commands, we force it via specific
    environment variables if stderr supports colors. *)
 let setup_env_for_colors = lazy(
-  if Lazy.force stderr_supports_colors then begin
+  if !Clflags.capture_outputs && Lazy.force stderr_supports_colors then begin
     let set var value =
       match Sys.getenv var with
       | exception Not_found -> Unix.putenv var value

src/clflags.ml

@@ -8,4 +8,4 @@ let debug_dep_path = ref false
 let dev_mode = ref false
 let workspace_root = ref "."
 let external_lib_deps_hint = ref []
-
+let capture_outputs = ref true

src/clflags.mli

@@ -29,3 +29,6 @@ val workspace_root : string ref
 
 (** The command line for "Hint: try: jbuilder external-lib-deps ..." *)
 val external_lib_deps_hint : string list ref
+
+(** Capture the output of sub-commands *)
+val capture_outputs : bool ref

src/context.ml

@@ -264,7 +264,8 @@ let create ~(kind : Kind.t) ~path ~base_env ~env_extra ~name ~merlin ~use_findli
        OCAML_COLOR is not supported so we use OCAMLPARAM. OCaml 4.02 doesn't support
        'color' in OCAMLPARAM, so we just don't force colors with 4.02. *)
     let ocaml_version = Scanf.sscanf version "%u.%u" (fun a b -> a, b) in
-    if Lazy.force Ansi_color.stderr_supports_colors
+    if !Clflags.capture_outputs
+    && Lazy.force Ansi_color.stderr_supports_colors
     && ocaml_version > (4, 02)
     && ocaml_version < (4, 05) then
       let value =

src/future.ml

@@ -619,16 +619,17 @@ module Scheduler = struct
           Format.eprintf "@{<kwd>Running@}[@{<id>%d@}]: %s@." id
             (Ansi_color.strip_colors_for_stderr command_line);
         let argv = Array.of_list (job.prog :: job.args) in
-        let output_filename, output_fd =
+        let output_filename, stdout_fd, stderr_fd, to_close =
           match job.stdout_to, job.stderr_to with
-          | Terminal, _ | _, Terminal ->
+          | (Terminal, _ | _, Terminal) when !Clflags.capture_outputs ->
             let fn = Temp.create "jbuilder" ".output" in
-            (Some fn, Unix.openfile fn [O_WRONLY; O_SHARE_DELETE] 0)
+            let fd = Unix.openfile fn [O_WRONLY; O_SHARE_DELETE] 0 in
+            (Some fn, fd, fd, Some fd)
           | _ ->
-            (None, Unix.stdin)
+            (None, Unix.stdout, Unix.stderr, None)
         in
-        let stdout, close_stdout = get_std_output job.stdout_to ~default:output_fd in
+        let stdout, close_stdout = get_std_output job.stdout_to ~default:stdout_fd in
-        let stderr, close_stderr = get_std_output job.stderr_to ~default:output_fd in
+        let stderr, close_stderr = get_std_output job.stderr_to ~default:stderr_fd in
         Option.iter job.dir ~f:(fun dir -> Sys.chdir dir);
         let pid =
           match job.env with
@@ -640,7 +641,7 @@ module Scheduler = struct
               Unix.stdin stdout stderr
         in
         Option.iter job.dir ~f:(fun _ -> Sys.chdir cwd);
-        if Option.is_some output_filename then Unix.close output_fd;
+        Option.iter to_close ~f:Unix.close;
         close_std_output close_stdout;
         close_std_output close_stderr;
         Running_jobs.add