diff --git a/tests/matplotlibrc b/tests/matplotlibrc
index 1abfc5ef56..cafb00a02b 100644
--- a/tests/matplotlibrc
+++ b/tests/matplotlibrc
@@ -1,4 +1,3 @@
 #### MATPLOTLIBRC FORMAT
 
 backend : Agg
-mathtext.fontset : cm
diff --git a/yt/visualization/plot_container.py b/yt/visualization/plot_container.py
index 062b1ed57b..6daf8c7fc1 100644
--- a/yt/visualization/plot_container.py
+++ b/yt/visualization/plot_container.py
@@ -2,12 +2,13 @@
 import builtins
 import os
 import textwrap
-from collections import defaultdict
+from collections import ChainMap, defaultdict
 from functools import wraps
 
 import matplotlib
 import numpy as np
 from matplotlib.cm import get_cmap
+from matplotlib.font_manager import FontProperties
 from more_itertools.more import always_iterable
 
 from yt._maintenance.deprecation import issue_deprecation_warning
@@ -236,6 +237,16 @@ def __init__(self, data_source, default_factory=None):
         return defaultdict.__init__(self, default_factory)
 
 
+_DEFAULT_FONT_PROPERTIES = {
+    "family": "stixgeneral",
+    "math_fontfamily": "cm",
+    "size": 18,
+    "fname": os.path.join(
+        matplotlib.get_data_path(), "fonts", "ttf", "STIXGeneral.ttf"
+    ),
+}
+
+
 class PlotContainer:
     """A container for generic plots"""
 
@@ -248,8 +259,6 @@ class PlotContainer:
     _units_config: dict
 
     def __init__(self, data_source, figure_size, fontsize):
-        from matplotlib.font_manager import FontProperties
-
         self.data_source = data_source
         self.ds = data_source.ds
         self.ts = self._initialize_dataset(self.ds)
@@ -257,10 +266,10 @@ def __init__(self, data_source, figure_size, fontsize):
             self.figure_size = float(figure_size[0]), float(figure_size[1])
         else:
             self.figure_size = float(figure_size)
-        font_path = os.path.join(
-            matplotlib.get_data_path(), "fonts", "ttf", "STIXGeneral.ttf"
-        )
-        self._font_properties = FontProperties(size=fontsize, fname=font_path)
+
+        font_props = _DEFAULT_FONT_PROPERTIES.copy()
+        font_props["size"] = fontsize
+        self._font_properties = FontProperties(**font_props)
         self._font_color = None
         self._xlabel = None
         self._ylabel = None
@@ -508,7 +517,6 @@ def set_font(self, font_dict=None):
         ... )
 
         """
-        from matplotlib.font_manager import FontProperties
 
         if font_dict is None:
             font_dict = {}
@@ -516,9 +524,8 @@ def set_font(self, font_dict=None):
             self._font_color = font_dict.pop("color")
         # Set default values if the user does not explicitly set them.
         # this prevents reverting to the matplotlib defaults.
-        font_dict.setdefault("family", "stixgeneral")
-        font_dict.setdefault("size", 18)
-        self._font_properties = FontProperties(**font_dict)
+        font_props = ChainMap(font_dict, _DEFAULT_FONT_PROPERTIES)
+        self._font_properties = FontProperties(**font_props)
         return self
 
     def set_font_size(self, size):