Use seed to randomize require order of test files

lib/ex_unit/lib/ex_unit.ex

@@ -395,8 +395,9 @@ defmodule ExUnit do
   @doc since: "1.12.0"
   @spec async_run() :: Task.t()
   def async_run() do
+    options = persist_defaults(configuration())
+
     Task.async(fn ->
-      options = persist_defaults(configuration())
       ExUnit.Runner.run(options, nil)
     end)
   end

lib/mix/lib/mix/compilers/test.ex

@@ -15,6 +15,7 @@ defmodule Mix.Compilers.Test do
   @compile {:no_warn_undefined, ExUnit}
   @stale_manifest "compile.test_stale"
   @manifest_vsn 1
+  @rand_algorithm :exs1024
 
   @doc """
   Requires and runs test files.
@@ -65,6 +66,8 @@ defmodule Mix.Compilers.Test do
       true ->
         task = ExUnit.async_run()
         warnings_as_errors? = Keyword.get(opts, :warnings_as_errors, false)
+        seed = Application.fetch_env!(:ex_unit, :seed)
+        test_files = shuffle(seed, test_files)
 
         try do
           failed? =
@@ -315,4 +318,13 @@ defmodule Mix.Compilers.Test do
   defp get_external_resources(module, cwd) do
     for file <- Module.get_attribute(module, :external_resource), do: Path.relative_to(file, cwd)
   end
+
+  defp shuffle(_seed = 0, list) do
+    list
+  end
+
+  defp shuffle(seed, list) do
+    _ = :rand.seed(@rand_algorithm, {seed, seed, seed})
+    Enum.shuffle(list)
+  end
 end

lib/mix/test/mix/tasks/test_test.exs

@@ -57,6 +57,32 @@ defmodule Mix.Tasks.TestTest do
     end
   end
 
+  describe "--seed" do
+    test "requires modules alphabetical order if seed is 0" do
+      in_fixture("test_stale", fn ->
+        opts = ["--seed", "0", "--trace"]
+        # Forcing the parallel compiler to operate synchronously
+        envs = [{"ELIXIR_ERL_OPTIONS", "+S 1:1"}]
+        output = mix(["test" | opts], envs)
+
+        matches = Regex.scan(~r"\wTest\s\[.*\]", output)
+        assert matches == [["BTest [test/b_test_stale.exs]"], ["ATest [test/a_test_stale.exs]"]]
+      end)
+    end
+
+    test "requires modules in random order if seed is provided" do
+      in_fixture("test_stale", fn ->
+        opts = ["--seed", "5", "--trace"]
+        # Forcing the parallel compiler to operate synchronously
+        envs = [{"ELIXIR_ERL_OPTIONS", "+S 1:1"}]
+        output = mix(["test" | opts], envs)
+
+        matches = Regex.scan(~r"\wTest\s\[.*\]", output)
+        assert matches == [["ATest [test/a_test_stale.exs]"], ["BTest [test/b_test_stale.exs]"]]
+      end)
+    end
+  end
+
   describe "--stale" do
     test "runs all tests for first run, then none on second" do
       in_fixture("test_stale", fn ->