diff --git a/changelogs/fragments/1486-fix-list-as-set.yml b/changelogs/fragments/1486-fix-list-as-set.yml
new file mode 100644
index 00000000..12cc1515
--- /dev/null
+++ b/changelogs/fragments/1486-fix-list-as-set.yml
@@ -0,0 +1,3 @@
+---
+minor_changes:
+  - Take a more generic approach for set comparison. Other models have object_types too
diff --git a/plugins/module_utils/netbox_users.py b/plugins/module_utils/netbox_users.py
index d8d2880d..3786e035 100644
--- a/plugins/module_utils/netbox_users.py
+++ b/plugins/module_utils/netbox_users.py
@@ -15,9 +15,6 @@
 NB_TOKENS = "tokens"
 NB_USERS = "users"
 
-# These suboptions are lists, but need to be modeled as sets for comparison purposes.
-LIST_AS_SET_KEYS = set(["permissions", "groups", "actions", "object_types"])
-
 
 class NetboxUsersModule(NetboxModule):
     def __init__(self, module, endpoint):
@@ -73,27 +70,18 @@ def run(self):
         self.module.exit_json(**self.result)
 
     def _update_netbox_object(self, data):
-        if self.endpoint == NB_TOKENS:
-            return self._update_netbox_token(data)
+        if self.endpoint == "users":
+            return self._update_netbox_user(data)
         else:
-            return self.__update_netbox_object__(data)
-
-    def _update_netbox_token(self, data):
-        if "key" in data:
-            del data["key"]
-        return self.__update_netbox_object__(data)
+            if self.endpoint == "tokens" and "key" in data:
+                del data["key"]
+            return super()._update_netbox_object(data)
 
-    def __update_netbox_object__(self, data):
+    def _update_netbox_user(self, data):
         serialized_nb_obj = self.nb_object.serialize()
         updated_obj = serialized_nb_obj.copy()
         updated_obj.update(data)
 
-        if serialized_nb_obj:
-            for key in LIST_AS_SET_KEYS:
-                if serialized_nb_obj.get(key) and data.get(key):
-                    serialized_nb_obj[key] = set(serialized_nb_obj[key])
-                    updated_obj[key] = set(data[key])
-
         if serialized_nb_obj == updated_obj:
             return serialized_nb_obj, None
         else:
diff --git a/plugins/module_utils/netbox_utils.py b/plugins/module_utils/netbox_utils.py
index 12a3f068..a73e33a7 100644
--- a/plugins/module_utils/netbox_utils.py
+++ b/plugins/module_utils/netbox_utils.py
@@ -737,6 +737,68 @@
     "virtualization.clustergroup": "cluster_groups",
 }
 
+# keys (for all endpoints) that should be converted from list to set
+LIST_AS_SET_KEYS = set(["tags"])
+
+# keys (for given endpoints) that should be converted from list to set
+LIST_AS_SET_ENDPOINT_KEYS = {
+    # tenancy
+    "contacts": set(["contact_groups"]),
+    # extra
+    "config_contexts": set(
+        [
+            "regions",
+            "site_groups",
+            "sites",
+            "roles",
+            "device_types",
+            "platforms",
+            "cluster_types",
+            "cluster_groups",
+            "clusters",
+            "tenant_groups",
+            "tenants",
+        ]
+    ),
+    "custom_fields": set(["object_types"]),
+    "custom_links": set(["object_types"]),
+    "export_templates": set(["object_types"]),
+    # users
+    "groups": set(["permissions"]),
+    "permissions": set(
+        [
+            "actions",
+            "object_types",
+        ]
+    ),
+    "users": set(
+        [
+            "groups",
+            "permissions",
+        ]
+    ),
+    # ipam
+    "l2vpns": set(
+        [
+            "import_targets",
+            "export_targets",
+        ]
+    ),
+    "services": set(["ports"]),
+    "service_templates": set(["ports"]),
+    "vlan_groups": set(["vid_ranges"]),
+    "vrfs": set(
+        [
+            "import_targets",
+            "export_targets",
+        ]
+    ),
+    # dcim, virtualization
+    "interfaces": set(["tagged_vlans"]),
+}
+
+config_context
+
 NETBOX_ARG_SPEC = dict(
     netbox_url=dict(type="str", required=True),
     netbox_token=dict(type="str", required=True, no_log=True),
@@ -1524,9 +1586,15 @@ def _update_netbox_object(self, data):
         updated_obj = serialized_nb_obj.copy()
         updated_obj.update(data)
 
-        if serialized_nb_obj.get("tags") and data.get("tags"):
-            serialized_nb_obj["tags"] = set(serialized_nb_obj["tags"])
-            updated_obj["tags"] = set(data["tags"])
+        # these fields are considerd a set and should be converted
+        for k in LIST_AS_SET_KEYS:
+            if serialized_nb_obj.get(k) and data.get(k):
+                serialized_nb_obj[k] = set(serialized_nb_obj[k])
+                updated_obj[k] = set(data[k])
+        for k in LIST_AS_SET_ENDPOINT_KEYS.get(self.endpoint, []):
+            if serialized_nb_obj.get(k) and data.get(k):
+                serialized_nb_obj[k] = set(serialized_nb_obj[k])
+                updated_obj[k] = set(data[k])
 
         # Ensure idempotency for site on older netbox versions
         version_pre_30 = self._version_check_greater("3.0", self.api_version)