diff --git a/symforce/opt/optimizer.py b/symforce/opt/optimizer.py
index 05603ce60..3d956a830 100644
--- a/symforce/opt/optimizer.py
+++ b/symforce/opt/optimizer.py
@@ -219,6 +219,11 @@ def __init__(
                 # We compute the linearization in the same order as `optimized_keys`
                 # so that e.g. columns of the generated jacobians are in the same order
                 factor_opt_keys = [opt_key for opt_key in optimized_keys if opt_key in factor.keys]
+                if not factor_opt_keys:
+                    raise ValueError(
+                        f"Factor {factor.name} has no arguments (keys: {factor.keys}) in "
+                        + f"optimized_keys ({optimized_keys})."
+                    )
                 numeric_factors.append(factor.to_numeric_factor(factor_opt_keys))
             else:
                 # Add unique keys to optimized keys
diff --git a/test/symforce_py_optimizer_test.py b/test/symforce_py_optimizer_test.py
index 596524dcf..721be57a8 100644
--- a/test/symforce_py_optimizer_test.py
+++ b/test/symforce_py_optimizer_test.py
@@ -13,6 +13,7 @@
 
 import symforce.symbolic as sf
 from symforce import logger
+from symforce import typing as T
 from symforce.opt.factor import Factor
 from symforce.opt.optimizer import Optimizer
 from symforce.test_util import TestCase
@@ -97,6 +98,26 @@ def prior_residual(x: sf.Rot3, epsilon: sf.Scalar, x_prior: sf.Rot3) -> sf.V3:
         index_entry2 = optimizer.linearization_index_entry("x1")
         self.assertEqual(index_entry, index_entry2)
 
+    def test_unoptimized_factor_exception(self) -> None:
+        """
+        Tests that a ValueError is raised if none of the factor keys match the optimizer keys.
+        """
+
+        def residual(x: T.Scalar) -> sf.V1:
+            return sf.V1((x - 2.71828) ** 2)
+
+        factor1 = Factor(["present_key"], residual)
+        factor2 = Factor(["absent_key"], residual)
+
+        optimized_keys = ["present_key", "other_present_key"]
+        with self.assertRaises(ValueError):
+            try:
+                Optimizer([factor1, factor2], optimized_keys)
+            except ValueError as err:
+                self.assertIn(str(factor2.keys), str(err))
+                self.assertIn(str(optimized_keys), str(err))
+                raise err from None
+
 
 if __name__ == "__main__":
     TestCase.main()