From 96df587fac35875326ff511c47593bc0cabd25eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 14 Feb 2018 13:19:23 +0100 Subject: [PATCH] introduce user:sync [--uid foo|--seenOnly|--showCount] --- core/Command/User/SyncBackend.php | 158 +++++++++++++++----- core/Migrations/Version20170221114437.php | 4 +- lib/private/User/AccountMapper.php | 32 ++++ lib/private/User/Sync/AllUsersIterator.php | 48 ++++++ lib/private/User/Sync/SeenUsersIterator.php | 53 +++++++ lib/private/User/Sync/UsersIterator.php | 53 +++++++ lib/private/User/SyncService.php | 50 +++---- tests/lib/User/SyncServiceTest.php | 3 +- 8 files changed, 330 insertions(+), 71 deletions(-) create mode 100644 lib/private/User/Sync/AllUsersIterator.php create mode 100644 lib/private/User/Sync/SeenUsersIterator.php create mode 100644 lib/private/User/Sync/UsersIterator.php diff --git a/core/Command/User/SyncBackend.php b/core/Command/User/SyncBackend.php index 2f1c8e4f76e2..930088695969 100644 --- a/core/Command/User/SyncBackend.php +++ b/core/Command/User/SyncBackend.php @@ -1,5 +1,6 @@ * @author Thomas Müller * * @copyright Copyright (c) 2018, ownCloud GmbH @@ -23,6 +24,8 @@ use OC\User\AccountMapper; +use OC\User\Sync\AllUsersIterator; +use OC\User\Sync\SeenUsersIterator; use OC\User\SyncService; use OCP\IConfig; use OCP\ILogger; @@ -80,6 +83,24 @@ protected function configure() { InputOption::VALUE_NONE, 'List all known backend classes' ) + ->addOption( + 'uid', + 'u', + InputOption::VALUE_REQUIRED, + 'sync only the user with the given user id' + ) + ->addOption( + 'seenOnly', + 's', + InputOption::VALUE_NONE, + 'sync only seen users' + ) + ->addOption( + 'showCount', + 'c', + InputOption::VALUE_NONE, + 'calculate user count before syncing' + ) ->addOption( 'missing-account-action', 'm', @@ -97,12 +118,12 @@ protected function execute(InputInterface $input, OutputInterface $output) { return 0; } $backendClassName = $input->getArgument('backend-class'); - if (is_null($backendClassName)) { - $output->writeln("No backend class name given. Please run ./occ help user:sync to understand how this command works."); + if ($backendClassName === null) { + $output->writeln('No backend class name given. Please run ./occ help user:sync to understand how this command works.'); return 1; } $backend = $this->getBackend($backendClassName); - if (is_null($backend)) { + if ($backend === null) { $output->writeln("The backend <$backendClassName> does not exist. Did you miss to enable the app?"); return 1; } @@ -116,7 +137,7 @@ protected function execute(InputInterface $input, OutputInterface $output) { if ($input->getOption('missing-account-action') !== null) { $missingAccountsAction = $input->getOption('missing-account-action'); if (!in_array($missingAccountsAction, $validActions, true)) { - $output->writeln("Unknown action. Choose between \"disable\" or \"remove\""); + $output->writeln('Unknown action. Choose between "disable" or "remove"'); return 1; } } else { @@ -132,27 +153,91 @@ protected function execute(InputInterface $input, OutputInterface $output) { $syncService = new SyncService($this->config, $this->logger, $this->accountMapper); - // analyse unknown users - $this->handleUnknownUsers($input, $output, $backend, $syncService, $missingAccountsAction, $validActions); + $uid = $input->getOption('uid'); + + if ($uid) { + $this->syncSingleUser($input, $output, $syncService, $backend, $uid, $missingAccountsAction, $validActions); + } else { + $this->syncMultipleUsers($input, $output, $syncService, $backend, $missingAccountsAction, $validActions); + } + + return 0; + } - // insert/update known users - $output->writeln("Insert new and update existing users ..."); + + /** + * @param InputInterface $input + * @param OutputInterface $output + * @param SyncService $syncService + * @param UserInterface $backend + * @param string $missingAccountsAction + * @param array $validActions + */ + private function syncMultipleUsers ( + InputInterface $input, + OutputInterface $output, + SyncService $syncService, + UserInterface $backend, + $missingAccountsAction, + array $validActions + ) { + $output->writeln('Analyse unknown users ...'); + $p = new ProgressBar($output); + $unknownUsers = $syncService->getNoLongerExistingUsers($backend, function () use ($p) { + $p->advance(); + }); + $p->finish(); + $output->writeln(''); + $output->writeln(''); + $this->handleUnknownUsers($unknownUsers, $input, $output, $missingAccountsAction, $validActions); + + $output->writeln('Insert new and update existing users ...'); $p = new ProgressBar($output); $max = null; - if ($backend->implementsActions(\OC_User_Backend::COUNT_USERS)) { + if ($backend->implementsActions(\OC_User_Backend::COUNT_USERS) && $input->hasOption('showCount')) { $max = $backend->countUsers(); } $p->start($max); - $syncService->run($backend, function () use ($p) { + + if ($input->hasOption('seenOnly')) { + $iterator = new SeenUsersIterator($this->accountMapper, get_class($backend)); + } else { + $iterator = new AllUsersIterator($backend); + } + $syncService->run($backend, $iterator, function () use ($p) { $p->advance(); }); $p->finish(); $output->writeln(''); $output->writeln(''); - - return 0; } + /** + * @param InputInterface $input + * @param OutputInterface $output + * @param SyncService $syncService + * @param UserInterface $backend + * @param string $uid + * @param string $missingAccountsAction + * @param array $validActions + */ + private function syncSingleUser( + InputInterface $input, + OutputInterface $output, + SyncService $syncService, + UserInterface $backend, + $uid, + $missingAccountsAction, + array $validActions + ) { + $output->writeln("Syncing $uid ..."); + if (!$backend->userExists($uid)) { + $this->handleUnknownUsers([$uid], $input, $output, $missingAccountsAction, $validActions); + } else { + // sync + $syncService->run($backend, new \ArrayIterator([$uid]), function (){}); + } + } /** * @param $backend * @return null|UserInterface @@ -187,58 +272,49 @@ private function doActionForAccountUids(array $uids, callable $callbackExists, c } /** + * @param string[] $unknownUsers * @param InputInterface $input * @param OutputInterface $output - * @param UserInterface $backend - * @param SyncService $syncService * @param $missingAccountsAction * @param $validActions */ - private function handleUnknownUsers(InputInterface $input, OutputInterface $output, UserInterface $backend, SyncService $syncService, $missingAccountsAction, $validActions) { - $output->writeln("Analyse unknown users ..."); - $p = new ProgressBar($output); - $toBeDeleted = $syncService->getNoLongerExistingUsers($backend, function () use ($p) { - $p->advance(); - }); - $p->finish(); - $output->writeln(''); - $output->writeln(''); + private function handleUnknownUsers(array $unknownUsers, InputInterface $input, OutputInterface $output, $missingAccountsAction, $validActions) { - if (empty($toBeDeleted)) { - $output->writeln("No unknown users have been detected."); + if (empty($unknownUsers)) { + $output->writeln('No unknown users have been detected.'); } else { - $output->writeln("Following users are no longer known with the connected backend."); + $output->writeln('Following users are no longer known with the connected backend.'); switch ($missingAccountsAction) { case 'disable': - $output->writeln("Proceeding to disable the accounts"); - $this->doActionForAccountUids($toBeDeleted, + $output->writeln('Proceeding to disable the accounts'); + $this->doActionForAccountUids($unknownUsers, function ($uid, IUser $ac) use ($output) { $ac->setEnabled(false); $output->writeln($uid); }, function ($uid) use ($output) { - $output->writeln($uid . " (unknown account for the user)"); + $output->writeln("$uid (unknown account for the user)"); }); break; case 'remove': - $output->writeln("Proceeding to remove the accounts"); - $this->doActionForAccountUids($toBeDeleted, + $output->writeln('Proceeding to remove the accounts'); + $this->doActionForAccountUids($unknownUsers, function ($uid, IUser $ac) use ($output) { $ac->delete(); $output->writeln($uid); }, function ($uid) use ($output) { - $output->writeln($uid . " (unknown account for the user)"); + $output->writeln("$uid (unknown account for the user)"); }); break; case 'ask later': - $output->writeln("listing the unknown accounts"); - $this->doActionForAccountUids($toBeDeleted, + $output->writeln('listing the unknown accounts'); + $this->doActionForAccountUids($unknownUsers, function ($uid) use ($output) { $output->writeln($uid); }, function ($uid) use ($output) { - $output->writeln($uid . " (unknown account for the user)"); + $output->writeln("$uid (unknown account for the user)"); }); // overwriting variables! $helper = $this->getHelper('question'); @@ -251,23 +327,23 @@ function ($uid) use ($output) { switch ($missingAccountsAction2) { // if "nothing" is selected, just ignore and finish case 'disable': - $output->writeln("Proceeding to disable the accounts"); - $this->doActionForAccountUids($toBeDeleted, + $output->writeln('Proceeding to disable the accounts'); + $this->doActionForAccountUids($unknownUsers, function ($uid, IUser $ac) { $ac->setEnabled(false); }, function ($uid) use ($output) { - $output->writeln($uid . " (unknown account for the user)"); + $output->writeln("$uid (unknown account for the user)"); }); break; case 'remove': - $output->writeln("Proceeding to remove the accounts"); - $this->doActionForAccountUids($toBeDeleted, + $output->writeln('Proceeding to remove the accounts'); + $this->doActionForAccountUids($unknownUsers, function ($uid, IUser $ac) { $ac->delete(); }, function ($uid) use ($output) { - $output->writeln($uid . " (unknown account for the user)"); + $output->writeln("$uid (unknown account for the user)"); }); break; } diff --git a/core/Migrations/Version20170221114437.php b/core/Migrations/Version20170221114437.php index 3f6d7df8fdab..c329fee1a66c 100644 --- a/core/Migrations/Version20170221114437.php +++ b/core/Migrations/Version20170221114437.php @@ -4,6 +4,7 @@ use OC\User\AccountMapper; use OC\User\AccountTermMapper; use OC\User\Database; +use OC\User\Sync\AllUsersIterator; use OC\User\SyncService; use OCP\Migration\ISimpleMigration; use OCP\Migration\IOutput; @@ -24,7 +25,8 @@ public function run(IOutput $out) { // insert/update known users $out->info("Insert new users ..."); $out->startProgress($backend->countUsers()); - $syncService->run($backend, function () use ($out) { + $backend-> + $syncService->run($backend, new AllUsersIterator($backend), function () use ($out) { $out->advance(); }); $out->finishProgress(); diff --git a/lib/private/User/AccountMapper.php b/lib/private/User/AccountMapper.php index 9a64bb6b912e..7b1bc0562c18 100644 --- a/lib/private/User/AccountMapper.php +++ b/lib/private/User/AccountMapper.php @@ -243,4 +243,36 @@ public function callForAllUsers($callback, $search, $onlySeen) { $stmt->closeCursor(); } + /** + * @param string $backend + * @param bool $hasLoggedIn + * @param integer $limit + * @param integer $offset + * @return string[] + */ + public function findUserIds($backend = null, $hasLoggedIn = null, $limit = null, $offset = null) { + $qb = $this->db->getQueryBuilder(); + $qb->select('user_id') + ->from($this->getTableName()); + if ($backend !== null) { + $qb->andWhere($qb->expr()->eq('backend', $qb->createNamedParameter($backend))); + } + if ($hasLoggedIn === true) { + $qb->andWhere($qb->expr()->gt('last_login', new Literal(0))); + } else if ($hasLoggedIn === false) { + $qb->andWhere($qb->expr()->eq('last_login', new Literal(0))); + } + if ($limit !== null) { + $qb->setMaxResults($limit); + } + if ($offset !== null) { + $qb->setFirstResult($offset); + } + + $stmt = $qb->execute(); + $rows = $stmt->fetchAll(); + $stmt->closeCursor(); + return $rows; + } + } diff --git a/lib/private/User/Sync/AllUsersIterator.php b/lib/private/User/Sync/AllUsersIterator.php new file mode 100644 index 000000000000..289cfe6c96c5 --- /dev/null +++ b/lib/private/User/Sync/AllUsersIterator.php @@ -0,0 +1,48 @@ + + * + * @copyright Copyright (c) 2018, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ +namespace OC\User\Sync; + +use OCP\UserInterface; + +class AllUsersIterator extends UsersIterator { + /** + * @var UserInterface + */ + private $backend; + + public function __construct(UserInterface $backend) { + $this->backend = $backend; + } + + public function rewind() { + parent::rewind(); + $this->data = $this->backend->getUsers('', self::LIMIT, 0); + } + + public function next() { + $this->position++; + if ($this->currentDataPos() === 0) { + $this->page++; + $offset = $this->page * self::LIMIT; + $this->data = $this->backend->getUsers('', self::LIMIT, $offset); + } + } +} \ No newline at end of file diff --git a/lib/private/User/Sync/SeenUsersIterator.php b/lib/private/User/Sync/SeenUsersIterator.php new file mode 100644 index 000000000000..ab5e61f222eb --- /dev/null +++ b/lib/private/User/Sync/SeenUsersIterator.php @@ -0,0 +1,53 @@ + + * + * @copyright Copyright (c) 2018, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ +namespace OC\User\Sync; + +use OC\User\AccountMapper; + +class SeenUsersIterator extends UsersIterator { + /** + * @var AccountMapper + */ + private $mapper; + /** + * @var string class name + */ + private $backend; + + public function __construct(AccountMapper $mapper, $backend) { + $this->mapper = $mapper; + $this->backend = $backend; + } + + public function rewind() { + parent::rewind(); + $this->data = $this->mapper->findUserIds($this->backend, true, self::LIMIT, 0); + } + + public function next() { + $this->position++; + if ($this->currentDataPos() === 0) { + $this->page++; + $offset = $this->page * self::LIMIT; + $this->data = $this->mapper->findUserIds($this->backend, true, self::LIMIT, $offset); + } + } +} \ No newline at end of file diff --git a/lib/private/User/Sync/UsersIterator.php b/lib/private/User/Sync/UsersIterator.php new file mode 100644 index 000000000000..a090e9d43af7 --- /dev/null +++ b/lib/private/User/Sync/UsersIterator.php @@ -0,0 +1,53 @@ + + * + * @copyright Copyright (c) 2018, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ +namespace OC\User\Sync; + +abstract class UsersIterator implements \Iterator { + protected $position = 0; + protected $page; + protected $data; + + const LIMIT = 500; + + + public function rewind() { + $this->position = 0; + $this->page = 0; + } + + public function current() { + return $this->data[$this->currentDataPos()]; + } + + public function key() { + return $this->position; + } + + public abstract function next(); + + public function valid() { + return isset($this->data[$this->currentDataPos()]); + } + + protected function currentDataPos() { + return $this->position % self::LIMIT; + } +} \ No newline at end of file diff --git a/lib/private/User/SyncService.php b/lib/private/User/SyncService.php index 9225e4752d1b..9306c03d33b7 100644 --- a/lib/private/User/SyncService.php +++ b/lib/private/User/SyncService.php @@ -1,5 +1,6 @@ * @author Thomas Müller * * @copyright Copyright (c) 2018, ownCloud GmbH @@ -95,33 +96,26 @@ public function getNoLongerExistingUsers(UserInterface $backend, \Closure $callb /** * @param UserInterface $backend to sync + * @param \Traversable $userIds of users * @param \Closure $callback is called for every user to progress display */ - public function run(UserInterface $backend, \Closure $callback) { - $limit = 500; - $offset = 0; - $backendClass = get_class($backend); - do { - $users = $backend->getUsers('', $limit, $offset); - - // update existing and insert new users - foreach ($users as $uid) { - try { - $account = $this->createOrSyncAccount($uid, $backend); - $uid = $account->getUserId(); // get correct case - // clean the user's preferences - $this->cleanPreferences($uid); - } catch (\Exception $e) { - // Error syncing this user - $this->logger->error("Error syncing user with uid: $uid and backend: {get_class($backend)}"); - $this->logger->logException($e); - } - - // call the callback - $callback($uid); + public function run(UserInterface $backend, \Traversable $userIds, \Closure $callback) { + // update existing and insert new users + foreach ($userIds as $uid) { + try { + $account = $this->createOrSyncAccount($uid, $backend); + $uid = $account->getUserId(); // get correct case + // clean the user's preferences + $this->cleanPreferences($uid); + } catch (\Exception $e) { + // Error syncing this user + $this->logger->error("Error syncing user with uid: $uid and backend: {get_class($backend)}"); + $this->logger->logException($e); } - $offset += $limit; - } while(count($users) >= $limit); + + // call the callback + $callback($uid); + } } /** @@ -237,10 +231,10 @@ private function syncHome(Account $a, UserInterface $backend) { if ($proividesHome) { $home = $backend->getHome($uid); } - if (!is_string($home) || substr($home, 0, 1) !== '/') { + if (!is_string($home) || $home[0] !== '/') { $home = $this->config->getSystemValue('datadirectory', \OC::$SERVERROOT . '/data') . "/$uid"; $this->logger->debug( - "User backend ".get_class($backend)." provided no home for <$uid>", + 'User backend ' .get_class($backend)." provided no home for <$uid>", ['app' => self::class] ); } @@ -281,7 +275,7 @@ private function syncSearchTerms(Account $a, UserInterface $backend) { $searchTerms = $backend->getSearchTerms($uid); $a->setSearchTerms($searchTerms); if ($a->haveTermsChanged()) { - $logTerms = join('|', $searchTerms); + $logTerms = implode('|', $searchTerms); $this->logger->debug( "Setting searchTerms for <$uid> to <$logTerms>", ['app' => self::class] ); @@ -367,7 +361,7 @@ public function createNewAccount($backend, $uid) { */ private function readUserConfig($uid, $app, $key) { $keys = $this->config->getUserKeys($uid, $app); - if (in_array($key, $keys)) { + if (in_array($key, $keys, true)) { $enabled = $this->config->getUserValue($uid, $app, $key); return [true, $enabled]; } diff --git a/tests/lib/User/SyncServiceTest.php b/tests/lib/User/SyncServiceTest.php index 32d59e99e330..0cb9ae4e3d23 100644 --- a/tests/lib/User/SyncServiceTest.php +++ b/tests/lib/User/SyncServiceTest.php @@ -26,6 +26,7 @@ use OC\User\Account; use OC\User\AccountMapper; use OC\User\Database; +use OC\User\Sync\AllUsersIterator; use OC\User\SyncService; use OCP\AppFramework\Db\DoesNotExistException; use OCP\IConfig; @@ -95,7 +96,7 @@ public function testSetupNewAccount() { $s = new SyncService($config, $logger, $mapper); - $s->run($backend, function($uid) {}); + $s->run($backend, new AllUsersIterator($backend), function($uid) {}); $this->invokePrivate($s, 'syncHome', [$account, $backend]); }