diff --git a/lib/private/Collaboration/Collaborators/UserPlugin.php b/lib/private/Collaboration/Collaborators/UserPlugin.php
index 6ff0c717145cc..c77469f12ce55 100644
--- a/lib/private/Collaboration/Collaborators/UserPlugin.php
+++ b/lib/private/Collaboration/Collaborators/UserPlugin.php
@@ -4,277 +4,156 @@
  * SPDX-FileCopyrightText: 2017 Nextcloud GmbH and Nextcloud contributors
  * SPDX-License-Identifier: AGPL-3.0-or-later
  */
+
 namespace OC\Collaboration\Collaborators;
 
-use OC\KnownUser\KnownUserService;
 use OCP\Collaboration\Collaborators\ISearchPlugin;
 use OCP\Collaboration\Collaborators\ISearchResult;
 use OCP\Collaboration\Collaborators\SearchResultType;
-use OCP\IConfig;
+use OCP\IAppConfig;
 use OCP\IGroupManager;
 use OCP\IUser;
 use OCP\IUserManager;
 use OCP\IUserSession;
 use OCP\Share\IShare;
 use OCP\UserStatus\IManager as IUserStatusManager;
+use OCP\UserStatus\IUserStatus;
 
-class UserPlugin implements ISearchPlugin {
-	protected bool $shareWithGroupOnly;
-
-	protected bool $shareeEnumeration;
-
-	protected bool $shareeEnumerationInGroupOnly;
-
-	protected bool $shareeEnumerationPhone;
-
-	protected bool $shareeEnumerationFullMatch;
-
-	protected bool $shareeEnumerationFullMatchUserId;
-
-	protected bool $shareeEnumerationfullMatchDisplayname;
-
-	protected bool $shareeEnumerationFullMatchEmail;
-
-	protected bool $shareeEnumerationFullMatchIgnoreSecondDisplayName;
-
+readonly class UserPlugin implements ISearchPlugin {
 	public function __construct(
-		private IConfig $config,
+		private IAppConfig $appConfig,
 		private IUserManager $userManager,
 		private IGroupManager $groupManager,
 		private IUserSession $userSession,
-		private KnownUserService $knownUserService,
 		private IUserStatusManager $userStatusManager,
-		private mixed $shareWithGroupOnlyExcludeGroupsList = [],
 	) {
-		$this->shareWithGroupOnly = $this->config->getAppValue('core', 'shareapi_only_share_with_group_members', 'no') === 'yes';
-		$this->shareeEnumeration = $this->config->getAppValue('core', 'shareapi_allow_share_dialog_user_enumeration', 'yes') === 'yes';
-		$this->shareeEnumerationInGroupOnly = $this->shareeEnumeration && $this->config->getAppValue('core', 'shareapi_restrict_user_enumeration_to_group', 'no') === 'yes';
-		$this->shareeEnumerationPhone = $this->shareeEnumeration && $this->config->getAppValue('core', 'shareapi_restrict_user_enumeration_to_phone', 'no') === 'yes';
-		$this->shareeEnumerationFullMatch = $this->config->getAppValue('core', 'shareapi_restrict_user_enumeration_full_match', 'yes') === 'yes';
-		$this->shareeEnumerationFullMatchUserId = $this->config->getAppValue('core', 'shareapi_restrict_user_enumeration_full_match_user_id', 'yes') === 'yes';
-		$this->shareeEnumerationfullMatchDisplayname = $this->config->getAppValue('core', 'shareapi_restrict_user_enumeration_full_match_displayname', 'yes') === 'yes';
-		$this->shareeEnumerationFullMatchEmail = $this->config->getAppValue('core', 'shareapi_restrict_user_enumeration_full_match_email', 'yes') === 'yes';
-		$this->shareeEnumerationFullMatchIgnoreSecondDisplayName = $this->config->getAppValue('core', 'shareapi_restrict_user_enumeration_full_match_ignore_second_dn', 'no') === 'yes';
-
-		if ($this->shareWithGroupOnly) {
-			$this->shareWithGroupOnlyExcludeGroupsList = json_decode($this->config->getAppValue('core', 'shareapi_only_share_with_group_members_exclude_group_list', ''), true) ?? [];
-		}
 	}
 
 	public function search($search, $limit, $offset, ISearchResult $searchResult): bool {
-		$result = ['wide' => [], 'exact' => []];
-		$users = [];
-		$hasMoreResults = false;
-
-		/** @var IUser */
+		/** @var IUser $currentUser */
 		$currentUser = $this->userSession->getUser();
-		$currentUserId = $currentUser->getUID();
-		$currentUserGroups = $this->groupManager->getUserGroupIds($currentUser);
 
-		// ShareWithGroupOnly filtering
-		$currentUserGroups = array_diff($currentUserGroups, $this->shareWithGroupOnlyExcludeGroupsList);
-
-		if ($this->shareWithGroupOnly || $this->shareeEnumerationInGroupOnly) {
-			// Search in all the groups this user is part of
-			foreach ($currentUserGroups as $userGroupId) {
-				$usersInGroup = $this->groupManager->displayNamesInGroup($userGroupId, $search, $limit, $offset);
-				foreach ($usersInGroup as $userId => $displayName) {
-					$userId = (string)$userId;
-					$user = $this->userManager->get($userId);
-					if (!$user?->isEnabled()) {
-						// Ignore disabled users
-						continue;
-					}
-					$users[$userId] = $user;
-				}
-				if (count($usersInGroup) >= $limit) {
-					$hasMoreResults = true;
-				}
-			}
-		}
+		/** @var array<string, array{0: 'wide'|'exact', 1: IUser}> $users */
+		$users = [];
 
-		// not limited to group only sharing
-		if (!$this->shareWithGroupOnly) {
-			if (!$this->shareeEnumerationPhone && !$this->shareeEnumerationInGroupOnly) {
-				// no restrictions, add everything
-				$usersTmp = $this->userManager->searchDisplayName($search, $limit, $offset);
-				foreach ($usersTmp as $user) {
-					if ($user->isEnabled()) { // Don't keep deactivated users
-						$users[$user->getUID()] = $user;
+		$shareeEnumeration = $this->appConfig->getValueString('core', 'shareapi_allow_share_dialog_user_enumeration', 'yes') === 'yes';
+		if ($shareeEnumeration) {
+			$shareWithGroupOnly = $this->appConfig->getValueString('core', 'shareapi_only_share_with_group_members', 'no') === 'yes';
+			$shareeEnumerationRestrictToGroup = $this->appConfig->getValueString('core', 'shareapi_restrict_user_enumeration_to_group', 'no') === 'yes';
+			$shareeEnumerationRestrictToPhone = $this->appConfig->getValueString('core', 'shareapi_restrict_user_enumeration_to_phone', 'no') === 'yes';
+
+			if (!$shareWithGroupOnly && !$shareeEnumerationRestrictToGroup && !$shareeEnumerationRestrictToPhone) {
+				// No restrictions, search everything.
+				$usersByDisplayName = $this->userManager->searchDisplayName($search, $limit, $offset);
+				foreach ($usersByDisplayName as $user) {
+					if ($user->isEnabled()) {
+						$users[$user->getUID()] = ['wide', $user];
 					}
 				}
 			} else {
-				// make sure to add phonebook matches if configured
-				if ($this->shareeEnumerationPhone) {
-					$usersTmp = $this->userManager->searchKnownUsersByDisplayName($currentUserId, $search, $limit, $offset);
-					foreach ($usersTmp as $user) {
-						if ($user->isEnabled()) { // Don't keep deactivated users
-							$users[$user->getUID()] = $user;
+				if ($shareWithGroupOnly || $shareeEnumerationRestrictToGroup) {
+					$shareWithGroupOnlyExcludeGroupsList = json_decode($this->appConfig->getValueString('core', 'shareapi_only_share_with_group_members_exclude_group_list', '[]'), true, 512, JSON_THROW_ON_ERROR) ?? [];
+					$allowedGroups = array_diff($this->groupManager->getUserGroupIds($currentUser), $shareWithGroupOnlyExcludeGroupsList);
+
+					foreach ($allowedGroups as $groupId) {
+						$usersInGroup = $this->groupManager->displayNamesInGroup($groupId, $search, $limit, $offset);
+						foreach ($usersInGroup as $userId => $displayName) {
+							$userId = (string)$userId;
+							$user = $this->userManager->get($userId);
+							if ($user !== null && $user->isEnabled()) {
+								$users[$userId] = ['wide', $user];
+							}
 						}
 					}
 				}
 
-				// additionally we need to add full matches
-				if ($this->shareeEnumerationFullMatch && $this->shareeEnumerationfullMatchDisplayname) {
-					$usersTmp = $this->userManager->searchDisplayName($search, $limit, $offset);
-					foreach ($usersTmp as $user) {
-						if ($user->isEnabled() && mb_strtolower($user->getDisplayName()) === mb_strtolower($search)) {
-							$users[$user->getUID()] = $user;
+				if ($shareeEnumerationRestrictToPhone) {
+					$usersInPhonebook = $this->userManager->searchKnownUsersByDisplayName($currentUser->getUID(), $search, $limit, $offset);
+					foreach ($usersInPhonebook as $user) {
+						if ($user->isEnabled()) {
+							$users[$user->getUID()] = ['wide', $user];
 						}
 					}
 				}
 			}
-
-			uasort($users, function (IUser $a, IUser $b) {
-				return strcasecmp($a->getDisplayName(), $b->getDisplayName());
-			});
 		}
 
-		$this->takeOutCurrentUser($users);
-
-		if (!$this->shareeEnumeration || count($users) < $limit) {
-			$hasMoreResults = true;
-		}
-
-		$foundUserById = false;
-		$lowerSearch = strtolower($search);
-		$userStatuses = $this->userStatusManager->getUserStatuses(array_keys($users));
-		foreach ($users as $uid => $user) {
-			$userDisplayName = $user->getDisplayName();
-			$userEmail = $user->getSystemEMailAddress();
-			$uid = (string)$uid;
-
-			$status = [];
-			if (array_key_exists($uid, $userStatuses)) {
-				$userStatus = $userStatuses[$uid];
-				$status = [
-					'status' => $userStatus->getStatus(),
-					'message' => $userStatus->getMessage(),
-					'icon' => $userStatus->getIcon(),
-					'clearAt' => $userStatus->getClearAt()
-						? (int)$userStatus->getClearAt()->format('U')
-						: null,
-				];
-			}
+		// Even if normal sharee enumeration is not allowed, full matches are still allowed.
+		$shareeEnumerationFullMatch = $this->appConfig->getValueString('core', 'shareapi_restrict_user_enumeration_full_match', 'yes') === 'yes';
+		if ($shareeEnumerationFullMatch && $search !== '') {
+			$shareeEnumerationFullMatchUserId = $this->appConfig->getValueString('core', 'shareapi_restrict_user_enumeration_full_match_userid', 'yes') === 'yes';
+			$shareeEnumerationFullMatchEmail = $this->appConfig->getValueString('core', 'shareapi_restrict_user_enumeration_full_match_email', 'yes') === 'yes';
+			$shareeEnumerationFullMatchIgnoreSecondDisplayName = $this->appConfig->getValueString('core', 'shareapi_restrict_user_enumeration_full_match_ignore_second_dn', 'no') === 'yes';
 
+			$lowerSearch = mb_strtolower($search);
 
-			if (
-				$this->shareeEnumerationFullMatch
-				&& $lowerSearch !== ''
-				&& (
-					strtolower($uid) === $lowerSearch
-					|| ($this->shareeEnumerationfullMatchDisplayname && strtolower($userDisplayName) === $lowerSearch)
-					|| ($this->shareeEnumerationFullMatchIgnoreSecondDisplayName && trim(strtolower(preg_replace('/ \(.*\)$/', '', $userDisplayName))) === $lowerSearch)
-					|| ($this->shareeEnumerationFullMatchEmail && strtolower($userEmail ?? '') === $lowerSearch)
-				)
-			) {
-				if (strtolower($uid) === $lowerSearch) {
-					$foundUserById = true;
-				}
-				$result['exact'][] = [
-					'label' => $userDisplayName,
-					'subline' => $status['message'] ?? '',
-					'icon' => 'icon-user',
-					'value' => [
-						'shareType' => IShare::TYPE_USER,
-						'shareWith' => $uid,
-					],
-					'shareWithDisplayNameUnique' => !empty($userEmail) ? $userEmail : $uid,
-					'status' => $status,
-				];
-			} else {
-				$addToWideResults = false;
-				if ($this->shareeEnumeration
-					&& !($this->shareeEnumerationInGroupOnly || $this->shareeEnumerationPhone)) {
-					$addToWideResults = true;
+			// Re-use the results from earlier if possible
+			$usersByDisplayName ??= $this->userManager->searchDisplayName($search, $limit, $offset);
+			foreach ($usersByDisplayName as $user) {
+				if ($user->isEnabled() && (mb_strtolower($user->getDisplayName()) === $lowerSearch || ($shareeEnumerationFullMatchIgnoreSecondDisplayName && trim(mb_strtolower(preg_replace('/ \(.*\)$/', '', $user->getDisplayName()))) === $lowerSearch))) {
+					$users[$user->getUID()] = ['exact', $user];
 				}
+			}
 
-				if ($this->shareeEnumerationPhone && $this->knownUserService->isKnownToUser($currentUserId, $user->getUID())) {
-					$addToWideResults = true;
+			if ($shareeEnumerationFullMatchUserId) {
+				$user = $this->userManager->get($search);
+				if ($user !== null) {
+					$users[$user->getUID()] = ['exact', $user];
 				}
+			}
 
-				if (!$addToWideResults && $this->shareeEnumerationInGroupOnly) {
-					$commonGroups = array_intersect($currentUserGroups, $this->groupManager->getUserGroupIds($user));
-					if (!empty($commonGroups)) {
-						$addToWideResults = true;
+			if ($shareeEnumerationFullMatchEmail) {
+				// TODO: Defer search to the manager/backends
+				$this->userManager->callForAllUsers(function (IUser $user) use (&$users, $lowerSearch): void {
+					$email = $user->getSystemEMailAddress();
+					if ($email !== null && mb_strtolower($email) === $lowerSearch) {
+						$users[$user->getUID()] = ['exact', $user];
 					}
-				}
-
-				if ($addToWideResults) {
-					$result['wide'][] = [
-						'label' => $userDisplayName,
-						'subline' => $status['message'] ?? '',
-						'icon' => 'icon-user',
-						'value' => [
-							'shareType' => IShare::TYPE_USER,
-							'shareWith' => $uid,
-						],
-						'shareWithDisplayNameUnique' => !empty($userEmail) ? $userEmail : $uid,
-						'status' => $status,
-					];
-				}
+				});
 			}
 		}
 
-		if ($this->shareeEnumerationFullMatch && $this->shareeEnumerationFullMatchUserId && $offset === 0 && !$foundUserById) {
-			// On page one we try if the search result has a direct hit on the
-			// user id and if so, we add that to the exact match list
-			$user = $this->userManager->get($search);
-			if ($user instanceof IUser) {
-				$addUser = true;
+		uasort($users, static fn (array $a, array $b): int => strcasecmp($a[1]->getDisplayName(), $b[1]->getDisplayName()));
 
-				if ($this->shareWithGroupOnly) {
-					// Only add, if we have a common group
-					$commonGroups = array_intersect($currentUserGroups, $this->groupManager->getUserGroupIds($user));
-					$addUser = !empty($commonGroups);
-				}
+		if (isset($users[$currentUser->getUID()])) {
+			unset($users[$currentUser->getUID()]);
+		}
 
-				if ($addUser) {
-					$status = [];
-					$uid = $user->getUID();
-					$userEmail = $user->getSystemEMailAddress();
-					if (array_key_exists($user->getUID(), $userStatuses)) {
-						$userStatus = $userStatuses[$user->getUID()];
-						$status = [
-							'status' => $userStatus->getStatus(),
-							'message' => $userStatus->getMessage(),
-							'icon' => $userStatus->getIcon(),
-							'clearAt' => $userStatus->getClearAt()
-								? (int)$userStatus->getClearAt()->format('U')
-								: null,
-						];
-					}
+		$userStatuses = array_map(
+			static fn (IUserStatus $userStatus) => [
+				'status' => $userStatus->getStatus(),
+				'message' => $userStatus->getMessage(),
+				'icon' => $userStatus->getIcon(),
+				'clearAt' => $userStatus->getClearAt()
+					? (int)$userStatus->getClearAt()->format('U')
+					: null,
+			],
+			$this->userStatusManager->getUserStatuses(array_keys($users)),
+		);
 
-					$result['exact'][] = [
-						'label' => $user->getDisplayName(),
-						'icon' => 'icon-user',
-						'subline' => $status['message'] ?? '',
-						'value' => [
-							'shareType' => IShare::TYPE_USER,
-							'shareWith' => $user->getUID(),
-						],
-						'shareWithDisplayNameUnique' => $userEmail !== null && $userEmail !== '' ? $userEmail : $uid,
-						'status' => $status,
-					];
-				}
-			}
+		$result = ['wide' => [], 'exact' => []];
+		foreach ($users as $match) {
+			[$type, $user] = $match;
+			$status = $userStatuses[$user->getUID()] ?? [];
+			$result[$type][] = [
+				'label' => $user->getDisplayName(),
+				'subline' => $status['message'] ?? '',
+				'icon' => 'icon-user',
+				'value' => [
+					'shareType' => IShare::TYPE_USER,
+					'shareWith' => $user->getUID(),
+				],
+				'shareWithDisplayNameUnique' => $user->getSystemEMailAddress() ?: $user->getUID(),
+				'status' => $status,
+			];
 		}
 
 		$type = new SearchResultType('users');
 		$searchResult->addResultSet($type, $result['wide'], $result['exact']);
-		if (count($result['exact'])) {
+		if ($result['exact'] !== []) {
 			$searchResult->markExactIdMatch($type);
 		}
 
-		return $hasMoreResults;
-	}
-
-	public function takeOutCurrentUser(array &$users): void {
-		$currentUser = $this->userSession->getUser();
-		if (!is_null($currentUser)) {
-			if (isset($users[$currentUser->getUID()])) {
-				unset($users[$currentUser->getUID()]);
-			}
-		}
+		return count($users) < $limit;
 	}
 }