diff --git a/src/jbuild.ml b/src/jbuild.ml
index a83d17d9..5966c99f 100644
--- a/src/jbuild.ml
+++ b/src/jbuild.ml
@@ -1066,10 +1066,14 @@ module Executables = struct
       (field_o "package" (junk >>> loc) >>= function
        | None -> return (t, None)
        | Some loc ->
-         Loc.warn loc
+         (Stanza.file_kind () >>| function
+         | Jbuild -> Loc.warn
+         | Dune   -> Loc.fail) >>= fun func ->
+         func loc
            "This field is useless without a (public_name%s ...) field."
            (if multi then "s" else "");
-         return (t, None))
+         return (t, None)
+      )
     | files ->
       Pkg.field >>= fun package ->
       return (t, Some { Install_conf. section = Bin; files; package })
diff --git a/test/blackbox-tests/test-cases/github992/run.t b/test/blackbox-tests/test-cases/github992/run.t
index 46553778..20a6f5fc 100644
--- a/test/blackbox-tests/test-cases/github992/run.t
+++ b/test/blackbox-tests/test-cases/github992/run.t
@@ -17,7 +17,8 @@ argument of "package".
 
   $ cd package-without-pub-name && dune build -p foo
   File "dune", line 3, characters 1-14:
-  Warning: This field is useless without a (public_name ...) field.
+  Error: This field is useless without a (public_name ...) field.
+  [1]
 
   $ cd package-without-pub-name-jbuild && dune build -p foo
   File "jbuild", line 3, characters 2-15: