diff --git a/test/config_test.exs b/test/config_test.exs
index 029273b9..d4a8531a 100644
--- a/test/config_test.exs
+++ b/test/config_test.exs
@@ -59,7 +59,7 @@ defmodule Sentry.ConfigTest do
     test "raises if not set" do
       assert_raise RuntimeError, ~r/:environment_name must be set/, fn ->
         modify_env(:sentry, environment_name: nil)
-        modify_system_env(%{"SENTRY_ENVIRONMENT" => nil})
+        delete_system_env("SENTRY_ENVIRONMENT")
         Config.environment_name()
       end
     end
diff --git a/test/support/test_environment_helper.ex b/test/support/test_environment_helper.ex
index d0996e94..78be83c2 100644
--- a/test/support/test_environment_helper.ex
+++ b/test/support/test_environment_helper.ex
@@ -41,4 +41,17 @@ defmodule Sentry.TestEnvironmentHelper do
       end)
     end)
   end
+
+  def delete_system_env(variable) do
+    original_env = System.fetch_env(variable)
+
+    System.delete_env(variable)
+
+    ExUnit.Callbacks.on_exit(fn ->
+      case original_env do
+        {:ok, val} -> System.put_env(variable, val)
+        :error -> :ok
+      end
+    end)
+  end
 end