Split up tests to reduce test time

The main change is to split them into modules to let `async: true` do
something. The second change is to use Erlang's process mailbox and
selective receive feature to fix message race and simplify code.

tests/heart_test/lib/heart.ex

@@ -63,16 +63,6 @@ defmodule Heart do
     send_message(server, <<@preparing_crash>>)
   end
 
-  @doc """
-  Wait timeout milliseconds for the next event
-
-  This is written trivial and only one caller process is supported at a time.
-  """
-  @spec next_event(GenServer.server(), non_neg_integer()) :: event() | :timeout
-  def next_event(server, timeout \\ 1000) do
-    GenServer.call(server, {:next_event, timeout}, timeout + 500)
-  end
-
   @impl GenServer
   def init(init_args) do
     shim = Application.app_dir(:heart_test, ["priv", "heart_fixture.so"]) |> Path.expand()
@@ -144,8 +134,7 @@ defmodule Heart do
        heart: heart_port,
        backend: backend_socket,
        requests: :queue.new(),
-       events: :queue.new(),
-       waiter: nil
+       notifications: init_args[:notifications]
      }}
   end
 
@@ -162,17 +151,6 @@ defmodule Heart do
     {:noreply, %{state | requests: :queue.in(from, state.requests)}}
   end
 
-  def handle_call({:next_event, timeout}, from, state) do
-    case :queue.out(state.events) do
-      {{:value, event}, new_events} ->
-        {:reply, event, %{state | events: new_events}}
-
-      {:empty, _calls} ->
-        timer_ref = Process.send_after(self(), {:timeout, from}, timeout)
-        {:noreply, %{state | waiter: {from, timer_ref}}}
-    end
-  end
-
   @impl GenServer
   def handle_info({heart, {:data, data}}, %{heart: heart} = state) do
     result = data |> IO.iodata_to_binary() |> decode_response()
@@ -195,32 +173,14 @@ defmodule Heart do
     {:noreply, process_event(state, {:event, data})}
   end
 
-  def handle_info({:timeout, client}, %{waiter: {client, _timer_ref}} = state) do
-    GenServer.reply(client, :timeout)
-    {:noreply, %{state | waiter: nil}}
-  end
-
-  def handle_info({:timeout, _client}, state) do
-    # Ignore stale timeout
-    {:noreply, state}
-  end
-
   def handle_info(message, state) do
     IO.puts("Got unexpected data #{inspect(message)}")
     {:noreply, state}
   end
 
   defp process_event(state, event) do
-    if state.waiter do
-      {client, timer_ref} = state.waiter
-
-      GenServer.reply(client, event)
-      _ = Process.cancel_timer(timer_ref)
-
-      %{state | waiter: nil}
-    else
-      %{state | events: :queue.in(event, state.events)}
-    end
+    send(state.notifications, event)
+    state
   end
 
   defp decode_response(<<@heart_ack>>), do: :heart_ack

tests/heart_test/test/basic_test.exs

@@ -0,0 +1,62 @@
+defmodule BasicTest do
+  use ExUnit.Case, async: true
+
+  import HeartTestCommon
+
+  setup do
+    common_setup()
+  end
+
+  test "heart acks on start and exits on shutdown", context do
+    heart = start_supervised!({Heart, context.init_args})
+    assert_receive {:heart, :heart_ack}
+    assert_receive {:event, "open(/dev/watchdog0) succeeded"}
+    assert_receive {:event, "pet(1)"}
+
+    graceful_shutdown(heart)
+  end
+
+  test "heart pets watchdog when petted itself", context do
+    heart = start_supervised!({Heart, context.init_args})
+    assert_receive {:heart, :heart_ack}
+    assert_receive {:event, "open(/dev/watchdog0) succeeded"}
+    assert_receive {:event, "pet(1)"}
+
+    Heart.pet(heart)
+    assert_receive {:event, "pet(1)"}
+
+    Heart.pet(heart)
+    assert_receive {:event, "pet(1)"}
+
+    graceful_shutdown(heart)
+  end
+
+  test "heart doesn't pet watchdog when not petted", context do
+    # The default wdt_timeout is 120s and the VM timeout is 60s, so no
+    # pet should happen. Wait for 6s to detect whether the default pet
+    # timeout of 5 seconds was erroneously used.
+    heart = start_supervised!({Heart, context.init_args})
+    assert_receive {:heart, :heart_ack}
+    assert_receive {:event, "open(/dev/watchdog0) succeeded"}
+    assert_receive {:event, "pet(1)"}
+
+    refute_receive _, 6000
+
+    graceful_shutdown(heart)
+  end
+
+  test "heart reboots when not petted", context do
+    # Shortest timeout is 11 seconds
+    start_supervised!({Heart, context.init_args ++ [heart_beat_timeout: 11]})
+    assert_receive {:heart, :heart_ack}
+    assert_receive {:event, "open(/dev/watchdog0) succeeded"}
+    assert_receive {:event, "pet(1)"}
+
+    Process.sleep(10000)
+    refute_received _
+
+    assert_receive {:event, "sync()"}, 1500
+    assert_receive {:event, "reboot(0x01234567)"}
+    assert_receive {:exit, 0}
+  end
+end

tests/heart_test/test/crash_dump_test.exs

@@ -0,0 +1,51 @@
+defmodule CrashDumpTest do
+  use ExUnit.Case, async: true
+
+  import HeartTestCommon
+
+  setup do
+    common_setup()
+  end
+
+  test "crash dump waits for notification", context do
+    heart = start_supervised!({Heart, context.init_args ++ [crash_dump_seconds: 10]})
+    assert_receive {:heart, :heart_ack}
+    assert_receive {:event, "open(/dev/watchdog0) succeeded"}
+    assert_receive {:event, "pet(1)"}
+
+    Heart.preparing_crash(heart)
+
+    # Capture the WDT pet that's sent before the crash
+    assert_receive {:event, "pet(1)"}
+
+    # Nothing should happen now
+    refute_receive _, 500
+
+    # Any write to the socket will cause a reboot now.
+    Heart.pet(heart)
+
+    assert_receive {:event, "sync()"}
+    assert_receive {:event, "reboot(0x01234567)"}
+    assert_receive {:exit, 0}
+  end
+
+  test "crash dump crashes anyway after timeout", context do
+    heart = start_supervised!({Heart, context.init_args ++ [crash_dump_seconds: 2]})
+    assert_receive {:heart, :heart_ack}
+    assert_receive {:event, "open(/dev/watchdog0) succeeded"}
+    assert_receive {:event, "pet(1)"}
+
+    Heart.preparing_crash(heart)
+
+    # Capture the WDT pet that's sent before the crash
+    assert_receive {:event, "pet(1)"}
+
+    # Nothing should happen for most of the 2 seconds
+    refute_receive _, 1900
+
+    # Timeout should trigger a crash in ~100 ms
+    assert_receive {:event, "sync()"}, 200
+    assert_receive {:event, "reboot(0x01234567)"}
+    assert_receive {:exit, 0}
+  end
+end

tests/heart_test/test/disable_hw_test.exs

@@ -0,0 +1,24 @@
+defmodule DisableHwTest do
+  use ExUnit.Case, async: true
+
+  import HeartTestCommon
+
+  setup do
+    common_setup()
+  end
+
+  test "sending disable_hw stops petting the hardware watchdog", context do
+    heart = start_supervised!({Heart, context.init_args})
+    assert_receive {:heart, :heart_ack}
+    assert_receive {:event, "open(/dev/watchdog0) succeeded"}
+    assert_receive {:event, "pet(1)"}
+
+    {:ok, :heart_ack} = Heart.set_cmd(heart, "disable_hw")
+
+    refute_receive _, 11000
+
+    # NOTE: even graceful shutdown doesn't do a final pet of the WDT
+    Heart.shutdown(heart)
+    assert_receive {:exit, 0}
+  end
+end

tests/heart_test/test/disable_vm_test.exs

@@ -0,0 +1,22 @@
+defmodule DisableVmTest do
+  use ExUnit.Case, async: true
+
+  import HeartTestCommon
+
+  setup do
+    common_setup()
+  end
+
+  test "sending disable_vm causes a heart timeout exit", context do
+    heart = start_supervised!({Heart, context.init_args})
+    assert_receive {:heart, :heart_ack}
+    assert_receive {:event, "open(/dev/watchdog0) succeeded"}
+    assert_receive {:event, "pet(1)"}
+
+    {:ok, :heart_ack} = Heart.set_cmd(heart, "disable_vm")
+
+    assert_receive {:event, "sync()"}
+    assert_receive {:event, "reboot(0x01234567)"}
+    assert_receive {:exit, 0}
+  end
+end

tests/heart_test/test/guarded_reboot_test.exs

@@ -0,0 +1,57 @@
+defmodule GuardedRebootTest do
+  use ExUnit.Case, async: true
+
+  import HeartTestCommon
+
+  setup do
+    common_setup()
+  end
+
+  test "guarded reboot", context do
+    heart = start_supervised!({Heart, context.init_args})
+    assert_receive {:heart, :heart_ack}
+    assert_receive {:event, "open(/dev/watchdog0) succeeded"}
+    assert_receive {:event, "pet(1)"}
+
+    {:ok, :heart_ack} = Heart.set_cmd(heart, "guarded_reboot")
+
+    # Final WDT pet
+    assert_receive {:event, "pet(1)"}
+
+    # Tell PID 1 to reboot
+    assert_receive {:event, "kill(1, SIGTERM)"}
+
+    # Proactive sync
+    assert_receive {:event, "sync()"}
+
+    Process.sleep(6)
+
+    # Run normal shutdown and check that there aren't any more WDT pets
+    Heart.shutdown(heart)
+    assert_receive {:exit, 0}
+  end
+
+  test "guarded poweroff", context do
+    heart = start_supervised!({Heart, context.init_args})
+    assert_receive {:heart, :heart_ack}
+    assert_receive {:event, "open(/dev/watchdog0) succeeded"}
+    assert_receive {:event, "pet(1)"}
+
+    {:ok, :heart_ack} = Heart.set_cmd(heart, "guarded_poweroff")
+
+    # Final WDT pet
+    assert_receive {:event, "pet(1)"}
+
+    # Tell PID 1 to reboot
+    assert_receive {:event, "kill(1, SIGUSR1)"}
+
+    # Proactive sync
+    assert_receive {:event, "sync()"}
+
+    Process.sleep(6)
+
+    # Run normal shutdown and check that there aren't any more WDT pets
+    Heart.shutdown(heart)
+    assert_receive {:exit, 0}
+  end
+end