test Mix task should exit with 1 when --warnings-as-errors opt is used

Previously some warnings were not detected as errors, exiting the command with 1.

An example is wronly naming a test file with the extension .ex instead of .exs

A test app has been uploaded to: https://github.com/eksperimental-debug/debug_elixir_test_wrong_ext

Previously:

    ❯ mix test --warnings-as-errors; echo $?
    Compiling 1 file (.ex)
    Generated debug_elixir_test_wrong_ext app
    warning: test/foo_test.ex does not match "*_test.exs" and won't be loaded
    Running ExUnit with seed: 194476, max_cases: 20

    ..
    Finished in 0.00 seconds (0.00s async, 0.00s sync)
    1 doctest, 1 test, 0 failures
    0

Now:

    ❯ mix test --warnings-as-errors; echo $?
    Compiling 1 file (.ex)
    Generated debug_elixir_test_wrong_ext app
    warning: test/foo_test.ex does not match "*_test.exs" and won't be loaded
    Running ExUnit with seed: 560339, max_cases: 20

    ..
    Finished in 0.00 seconds (0.00s async, 0.00s sync)
    1 doctest, 1 test, 0 failures

    ERROR! Test suite aborted after successful execution due to warnings while using the --warnings-as-errors option
    1

All tests will be executed, the warning is printed, but it prints an error message and exits with 1.

lib/mix/lib/mix/tasks/test.ex

@@ -608,7 +608,25 @@ defmodule Mix.Tasks.Test do
       |> filter_to_allowed_files(allowed_files)
       |> filter_by_partition(shell, partitions)
 
-    display_warn_test_pattern(test_files, test_pattern, unfiltered_test_files, warn_test_pattern)
+    warnings =
+      display_warn_test_pattern(
+        test_files,
+        test_pattern,
+        unfiltered_test_files,
+        warn_test_pattern
+      )
+
+    if warnings != [] and Keyword.get(opts, :warnings_as_errors) do
+      System.at_exit(fn _ ->
+        message =
+          "\nERROR! Test suite aborted after successful execution due to " <>
+            "warnings while using the --warnings-as-errors option"
+
+        Mix.shell().error(message)
+
+        exit({:shutdown, 1})
+      end)
+    end
 
     try do
       Enum.each(test_paths, &require_test_helper(shell, &1))

lib/mix/test/mix/tasks/compile.erlang_test.exs

@@ -118,34 +118,36 @@ defmodule Mix.Tasks.Compile.ErlangTest do
       my_fn() -> ok.
       """)
 
-      capture_io(fn ->
-        assert {:ok, [diagnostic]} = Mix.Tasks.Compile.Erlang.run([])
-
-        assert %Mix.Task.Compiler.Diagnostic{
-                 file: ^source,
-                 source: ^source,
-                 compiler_name: "erl_lint",
-                 message: "function my_fn/0 is unused",
-                 position: position(2, 1),
-                 severity: :warning
-               } = diagnostic
-
-        # Should return warning without recompiling file
-        assert {:noop, [^diagnostic]} = Mix.Tasks.Compile.Erlang.run(["--verbose"])
-        refute_received {:mix_shell, :info, ["Compiled src/has_warning.erl"]}
-
-        assert [^diagnostic] = Mix.Tasks.Compile.Erlang.diagnostics()
-        assert [^diagnostic] = Mix.Task.Compiler.diagnostics()
-
-        # Should not return warning after changing file
-        File.write!(file, """
-        -module(has_warning).
-        -export([my_fn/0]).
-        my_fn() -> ok.
-        """)
-
-        ensure_touched(file)
-        assert {:ok, []} = Mix.Tasks.Compile.Erlang.run([])
+      capture_io(:stderr, fn ->
+        capture_io(fn ->
+          assert {:ok, [diagnostic]} = Mix.Tasks.Compile.Erlang.run([])
+
+          assert %Mix.Task.Compiler.Diagnostic{
+                   file: ^source,
+                   source: ^source,
+                   compiler_name: "erl_lint",
+                   message: "function my_fn/0 is unused",
+                   position: position(2, 1),
+                   severity: :warning
+                 } = diagnostic
+
+          # Should return warning without recompiling file
+          assert {:noop, [^diagnostic]} = Mix.Tasks.Compile.Erlang.run(["--verbose"])
+          refute_received {:mix_shell, :info, ["Compiled src/has_warning.erl"]}
+
+          assert [^diagnostic] = Mix.Tasks.Compile.Erlang.diagnostics()
+          assert [^diagnostic] = Mix.Task.Compiler.diagnostics()
+
+          # Should not return warning after changing file
+          File.write!(file, """
+          -module(has_warning).
+          -export([my_fn/0]).
+          my_fn() -> ok.
+          """)
+
+          ensure_touched(file)
+          assert {:ok, []} = Mix.Tasks.Compile.Erlang.run([])
+        end)
       end)
     end)
   end