diff --git a/changelog.d/6796.bugfix b/changelog.d/6796.bugfix
new file mode 100644
index 000000000000..206a15731151
--- /dev/null
+++ b/changelog.d/6796.bugfix
@@ -0,0 +1 @@
+Fix bug where querying a remote user's device keys that weren't cached resulted in only returning a single device.
diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py
index 2d889364d4bc..95a9d71f4112 100644
--- a/synapse/handlers/e2e_keys.py
+++ b/synapse/handlers/e2e_keys.py
@@ -208,8 +208,9 @@ def do_remote_query(destination):
                         )
 
                     user_devices = user_devices["devices"]
+                    user_results = results.setdefault(user_id, {})
                     for device in user_devices:
-                        results[user_id] = {device["device_id"]: device["keys"]}
+                        user_results[device["device_id"]] = device["keys"]
                     user_ids_updated.append(user_id)
                 except Exception as e:
                     failures[destination] = _exception_to_failure(e)