Fix KeyError in _rel_agnostic_fields_ref for uncommon rel.to setup

django-taggit manually sets up `rel.to` and therefore there is no "to"
kwarg.

Fixes #22263
Ref: jazzband/django-taggit#206

django/db/migrations/autodetector.py

@@ -71,7 +71,11 @@ def _rel_agnostic_fields_def(fields):
             for name, field in fields:
                 deconstruction = field.deconstruct()[1:]
                 if field.rel and field.rel.to:
-                    del deconstruction[2]['to']
+                    try:
+                        del deconstruction[2]['to']
+                    except KeyError:
+                        # `rel.to` does not have to come from kwargs.
+                        pass
                 fields_def.append(deconstruction)
             return fields_def