diff --git a/apps/federatedfilesharing/lib/Command/PollIncomingShares.php b/apps/federatedfilesharing/lib/Command/PollIncomingShares.php index f4b59e152b56..2145344dc071 100644 --- a/apps/federatedfilesharing/lib/Command/PollIncomingShares.php +++ b/apps/federatedfilesharing/lib/Command/PollIncomingShares.php @@ -84,6 +84,7 @@ protected function configure() { * @return int|null|void */ public function execute(InputInterface $input, OutputInterface $output) { + $output->writeln("WARNING: incoming-shares:poll has been deprecated and replaced by periodic external shares cronjob. Please check Federated Cloud Sharing settings and documentation."); if ($this->externalMountProvider === null) { $output->writeln("Polling is not possible when files_sharing app is disabled. Please enable it with 'occ app:enable files_sharing'"); return 1; diff --git a/apps/federatedfilesharing/lib/FederatedShareProvider.php b/apps/federatedfilesharing/lib/FederatedShareProvider.php index 1c4242b20915..9064bd0600f5 100644 --- a/apps/federatedfilesharing/lib/FederatedShareProvider.php +++ b/apps/federatedfilesharing/lib/FederatedShareProvider.php @@ -959,6 +959,16 @@ public function userDeletedFromGroup($uid, $gid) { return; } + /** + * check if scan of federated shares from other ownCloud instances should be performed + * + * @return bool + */ + public function isCronjobScanExternalEnabled() { + $result = $this->config->getAppValue('files_sharing', 'cronjob_scan_external_enabled', 'no'); + return ($result === 'yes') ? true : false; + } + /** * check if users from other ownCloud instances are allowed to mount public links share by this instance * diff --git a/apps/federatedfilesharing/lib/Panels/AdminPanel.php b/apps/federatedfilesharing/lib/Panels/AdminPanel.php index 238e615a2f26..9caf65bfbfa9 100644 --- a/apps/federatedfilesharing/lib/Panels/AdminPanel.php +++ b/apps/federatedfilesharing/lib/Panels/AdminPanel.php @@ -53,6 +53,7 @@ public function getSectionID() { public function getPanel() { $tmpl = new Template('federatedfilesharing', 'settings-admin'); + $tmpl->assign('cronjobScanExternalEnabled', $this->shareProvider->isCronjobScanExternalEnabled()); $tmpl->assign('outgoingServer2serverShareEnabled', $this->shareProvider->isOutgoingServer2serverShareEnabled()); $tmpl->assign('incomingServer2serverShareEnabled', $this->shareProvider->isIncomingServer2serverShareEnabled()); $tmpl->assign( diff --git a/apps/federatedfilesharing/templates/settings-admin.php b/apps/federatedfilesharing/templates/settings-admin.php index a774c1455404..4f7e4be32e8b 100644 --- a/apps/federatedfilesharing/templates/settings-admin.php +++ b/apps/federatedfilesharing/templates/settings-admin.php @@ -10,6 +10,16 @@ title="t('Open documentation'));?>" href=""> +

+ /> + +

+

dbConnection->method('getQueryBuilder')->willReturn($qbMock); $this->commandTester->execute([]); $output = $this->commandTester->getDisplay(); - $this->assertEmpty($output); + $this->assertEquals($output, "WARNING: incoming-shares:poll has been deprecated and replaced by periodic external shares cronjob. Please check Federated Cloud Sharing settings and documentation.\n"); } public function testWithFilesSharingDisabled() { diff --git a/apps/federatedfilesharing/tests/FederatedShareProviderTest.php b/apps/federatedfilesharing/tests/FederatedShareProviderTest.php index 6d86b0640750..a5285392fdbd 100644 --- a/apps/federatedfilesharing/tests/FederatedShareProviderTest.php +++ b/apps/federatedfilesharing/tests/FederatedShareProviderTest.php @@ -857,6 +857,22 @@ public function testDeleteUser($owner, $initiator, $recipient, $deletedUser, $ro $this->assertCount($rowDeleted ? 0 : 1, $data); } + /** + * @dataProvider dataTestFederatedSharingSettings + * + * @param string $isEnabled + * @param bool $expected + */ + public function testIsCronjobScanExternalEnabled($isEnabled, $expected) { + $this->config->expects($this->once())->method('getAppValue') + ->with('files_sharing', 'cronjob_scan_external_enabled', 'no') + ->willReturn($isEnabled); + + $this->assertSame($expected, + $this->provider->isCronjobScanExternalEnabled() + ); + } + /** * @dataProvider dataTestFederatedSharingSettings * diff --git a/apps/federatedfilesharing/tests/Panels/AdminPanelTest.php b/apps/federatedfilesharing/tests/Panels/AdminPanelTest.php index b073181cbc04..5b49debf6abe 100644 --- a/apps/federatedfilesharing/tests/Panels/AdminPanelTest.php +++ b/apps/federatedfilesharing/tests/Panels/AdminPanelTest.php @@ -55,6 +55,7 @@ public function testGetPriority() { } public function testGetPanel() { + $this->shareProvider->expects($this->once())->method('isCronjobScanExternalEnabled')->willReturn(false); $this->shareProvider->expects($this->once())->method('isOutgoingServer2serverShareEnabled')->willReturn(true); $this->shareProvider->expects($this->once())->method('isIncomingServer2serverShareEnabled')->willReturn(true); $templateHtml = $this->panel->getPanel()->fetchPage(); diff --git a/apps/files_sharing/appinfo/Migrations/Version20200504211654.php b/apps/files_sharing/appinfo/Migrations/Version20200504211654.php new file mode 100644 index 000000000000..a063ef04068d --- /dev/null +++ b/apps/files_sharing/appinfo/Migrations/Version20200504211654.php @@ -0,0 +1,31 @@ +hasTable("${prefix}share_external")) { + $table = $schema->getTable("${prefix}share_external"); + + if (!$table->hasColumn('lastscan')) { + $table->addColumn( + 'lastscan', + Types::BIGINT, + [ + 'length' => 11, + 'unsigned' => true, + 'notnull' => false, + ] + ); + } + } + } +} diff --git a/apps/files_sharing/appinfo/info.xml b/apps/files_sharing/appinfo/info.xml index d63eb8c63f1c..4f96a7747ed8 100644 --- a/apps/files_sharing/appinfo/info.xml +++ b/apps/files_sharing/appinfo/info.xml @@ -10,7 +10,7 @@ Turning the feature off removes shared files and folders on the server for all s AGPL Michael Gapczynski, Bjoern Schiessle - 0.12.0 + 0.13.0 @@ -35,6 +35,7 @@ Turning the feature off removes shared files and folders on the server for all s OCA\Files_Sharing\DeleteOrphanedSharesJob OCA\Files_Sharing\ExpireSharesJob + OCA\Files_Sharing\External\ScanExternalSharesJob diff --git a/apps/files_sharing/lib/External/ScanExternalSharesJob.php b/apps/files_sharing/lib/External/ScanExternalSharesJob.php new file mode 100644 index 000000000000..26b8647af75d --- /dev/null +++ b/apps/files_sharing/lib/External/ScanExternalSharesJob.php @@ -0,0 +1,274 @@ + + * + * @copyright Copyright (c) 2020, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OCA\Files_Sharing\External; + +use OC\User\NoUserException; +use OC\BackgroundJob\TimedJob; +use OCP\Files\Cache\IScanner; +use OCP\IConfig; +use OCP\IDBConnection; +use OCP\ILogger; +use OCP\IUserManager; +use OCP\Files\StorageNotAvailableException; + +/** + * Class ScanExternalShares is a background job used to run the external shares + * scanner over external shares that are eligible for scanning, + * to ensure integrity of the file cache. Scanner will search for external shares + * that satisfy the below requirements: + * - ensure that within single cron run, at max [cronjob_scan_external_batch] + * scans will be performed out of all accepted external shares + * - scan of that external share has not been performed within + * last [cronjob_scan_external_min_scan] seconds + * - user still exists, and has been active recently, meaning logged in at + * least [cronjob_scan_external_min_login] seconds ago + * - external share root etag/mtime changed, signaling that remote changed + * and requires scan + * + * @package OCA\Files_Sharing\External\BackgroundJob + */ +class ScanExternalSharesJob extends TimedJob { + /** @var IDBConnection */ + private $connection; + /** @var Manager */ + private $externalManager; + /** @var IConfig */ + private $config; + /** @var IUserManager */ + private $userManager; + /** @var ILogger */ + private $logger; + + const DEFAULT_MIN_LAST_SCAN = 3*60*60; + const DEFAULT_MIN_LOGIN = 24*60*60; + const DEFAULT_SHARES_PER_SESSION = 100; + const BATCH_SIZE = 10; + + public function __construct(IDBConnection $connection = null, + IConfig $config = null, + IUserManager $userManager = null, + ILogger $logger = null, + Manager $externalManager = null) { + // Run once per 10 minutes + $this->setInterval(60 * 10); + + if ($logger === null || $userManager === null || $config === null || $connection === null || $externalManager === null) { + $this->fixDIForJobs(); + } else { + $this->connection = $connection; + $this->externalManager = $externalManager; + $this->config = $config; + $this->userManager = $userManager; + $this->logger = $logger; + } + } + + protected function fixDIForJobs() { + $this->connection = \OC::$server->getDatabaseConnection(); + $this->config = \OC::$server->getConfig(); + $this->userManager = \OC::$server->getUserManager(); + $this->logger = \OC::$server->getLogger(); + $this->externalManager = new Manager( + $this->connection, + \OC\Files\Filesystem::getMountManager(), + \OC\Files\Filesystem::getLoader(), + \OC::$server->getNotificationManager(), + \OC::$server->getEventDispatcher(), + null + ); + } + + /** + * @param $argument + * @throws \Exception + */ + protected function run($argument) { + $enabled = $this->config->getAppValue('files_sharing', 'cronjob_scan_external_enabled', 'no'); + if ($enabled !== 'yes') { + $this->logger->debug( + "Fed share scanner disabled, ignoring the run" + ); + return; + } + + $lastLoginThreshold = $this->config->getAppValue('files_sharing', 'cronjob_scan_external_min_login', self::DEFAULT_MIN_LOGIN); + $lastScanThreshold = $this->config->getAppValue('files_sharing', 'cronjob_scan_external_min_scan', self::DEFAULT_MIN_LAST_SCAN); + $maxSharesPerSession = $this->config->getAppValue('files_sharing', 'cronjob_scan_external_batch', self::DEFAULT_SHARES_PER_SESSION); + $batchPerSession = \min($maxSharesPerSession, self::BATCH_SIZE); + + $scannedShares = 0; + + do { + $offset = $this->config->getAppValue('files_sharing', 'cronjob_scan_external_offset', 0); + $shares = $this->getAcceptedShares($batchPerSession, $offset); + + $searchedBatch = 0; + foreach ($shares as $share) { + if ($this->shouldScan($share, $lastLoginThreshold, $lastScanThreshold)) { + // make sure not to scan this share again within [cronjob_scan_external_min_scan] + $this->updateLastScanned($share['id'], \time()); + + // do scan share + $this->scan($share); + $scannedShares += 1; + } + $searchedBatch += 1; + } + + $this->logger->debug( + "Fed share scanner performed scan of $scannedShares/$searchedBatch shares at offset $offset" + ); + + if (\count($shares) < $batchPerSession) { + // next run wont have any users to scan, + // as we returned less than the limit + $offset = 0; + } else { + $offset += $batchPerSession; + } + + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_offset', $offset); + } while ($offset !== 0 && $scannedShares < $maxSharesPerSession); + } + + protected function shouldScan($share, $lastLoginThreshold, $lastScanThreshold) { + $now = \time(); + + // check last login + $user = $this->userManager->get($share['user']); + if ($user === null) { + $this->logger->debug( + "Will not scan fed share {$share['mountpoint']} for uid {$share['user']}, user does not exist" + ); + return false; + } + $lastLogin = $user->getLastLogin(); + if ($lastLogin + $lastLoginThreshold < $now) { + $this->logger->debug( + "Will not scan fed share {$share['mountpoint']} for uid {$share['user']}, user did not login in last $lastLoginThreshold seconds" + ); + return false; + } + + // check last scan + $lastScan = $share['lastscan'] ? \intval($share['lastscan']) : 0; + if ($lastScan + $lastScanThreshold > $now) { + $this->logger->debug( + "Will not scan fed share {$share['mountpoint']} for uid {$share['user']}, share has been already scanned in last $lastScanThreshold seconds" + ); + return false; + } + + return true; + } + + protected function scan($share) { + // get mount + $options = [ + 'remote' => $share['remote'], + 'token' => $share['share_token'], + 'password' => $share['password'], + 'mountpoint' => $share['mountpoint'], + 'owner' => $share['owner'] + ]; + + try { + // get mount + $this->externalManager->setUid($share['user']); + $mount = $this->externalManager->getMount($options); + $this->externalManager->setUid(null); + + // check if root storage updated + $storage = $mount->getStorage(); + $localMtime = $storage->filemtime(''); + $hasUpdated = $storage->hasUpdated('', $localMtime); + if (!$hasUpdated) { + $this->logger->debug( + "Scanned fed share {$share['mountpoint']} for uid {$share['user']}, share is up to date" + ); + return false; + } + + // scan recursive, and do not reuse anything + // as we need to force scanning of the external share storage + $propagator = $storage->getPropagator(); + $propagator->beginBatch(); + $storage->getScanner()->scan('', IScanner::SCAN_RECURSIVE, IScanner::REUSE_NONE); + $propagator->commitBatch(); + + $this->logger->debug( + "Scanned fed share {$share['mountpoint']} for uid {$share['user']} with last scan {$share['lastscan']}" + ); + } catch (NoUserException $e) { + // uid was null so we need to set it + $this->externalManager->setUid($share['user']); + $this->externalManager->removeShare($mount->getMountPoint()); + // and now we need to reset uid back to null + $this->externalManager->setUid(null); + $this->logger->debug( + "Remote {$share['remote']} reports that external share with {$share['mountpoint']} for uid {$share['user']} no longer exists. Removing it.." + ); + } catch (StorageNotAvailableException $e) { + $reason = $e->getMessage(); + $this->logger->debug( + "Skipping external share {$share['mountpoint']} for uid {$share['user']} from remote {$share['remote']} as share is unreachable. Reason: {$reason}" + ); + } catch (\Exception $e) { + $this->logger->debug( + "Skipping external share {$share['mountpoint']} for uid {$share['user']} from remote {$share['remote']} due to internal server error" + ); + $this->logger->logException($e, ['app' => 'federatedfilesharing']); + } + + $this->tearDownStorage(); + + return true; + } + + protected function tearDownStorage() { + \OC_Util::tearDownFS(); + } + + protected function getAcceptedShares($limit, $offset) { + $qb = $this->connection->getQueryBuilder(); + $qb->select('*') + ->from('share_external') + ->where($qb->expr()->eq('accepted', $qb->expr()->literal('1'))); + + $qb->setMaxResults($limit); + $qb->setFirstResult($offset); + $qb->orderBy('id'); + + $cursor = $qb->execute(); + return $cursor->fetchAll(); + } + + protected function updateLastScanned($id, $updatedTime) { + $qb = $this->connection->getQueryBuilder(); + $qb->update('share_external') + ->set('lastscan', $qb->createNamedParameter($updatedTime)) + ->where($qb->expr()->eq('id', $qb->createNamedParameter($id))); + + $result = $qb->execute(); + return $result === 1; + } +} diff --git a/apps/files_sharing/tests/External/ScanExternalSharesJobTest.php b/apps/files_sharing/tests/External/ScanExternalSharesJobTest.php new file mode 100644 index 000000000000..0c3dd4a2b2f8 --- /dev/null +++ b/apps/files_sharing/tests/External/ScanExternalSharesJobTest.php @@ -0,0 +1,458 @@ + + * + * @copyright Copyright (c) 2020, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OCA\Files_Sharing\Tests\External; + +use OC\Files\Cache\Propagator; +use OC\Files\Cache\Scanner; +use OC\User\NoUserException; +use OCA\Files_Sharing\External\Manager; +use OCA\Files_Sharing\External\Mount; +use OCA\Files_Sharing\External\ScanExternalSharesJob; +use OCA\Files_Sharing\External\Storage; +use OCP\Files\StorageNotAvailableException; +use OCP\IConfig; +use OCP\IDBConnection; +use OCP\ILogger; +use OCP\IUser; +use OCP\IUserManager; +use Test\TestCase; + +/** + * Class ScanFilesTest + * + * @group DB + * + * @package OCA\Files_Sharing\Tests\External + */ +class ScanExternalSharesJobTest extends TestCase { + + /** @var Manager */ + private $externalManager; + + /** @var IUserManager */ + private $userManager; + + /** @var IDBConnection */ + private $connection; + + /** @var IConfig */ + private $config; + + protected function setUp(): void { + parent::setUp(); + + $this->connection = \OC::$server->getDatabaseConnection(); + $this->config = \OC::$server->getConfig(); + $this->userManager = $this->createMock(IUserManager::class); + $this->externalManager = $this->createMock(Manager::class); + + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_enabled', 'yes'); + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_min_login', ScanExternalSharesJob::DEFAULT_MIN_LAST_SCAN); + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_min_scan', ScanExternalSharesJob::DEFAULT_MIN_LOGIN); + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_batch', ScanExternalSharesJob::DEFAULT_SHARES_PER_SESSION); + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_offset', 0); + + $shareExternalQuery = $this->connection->getQueryBuilder(); + $shareExternalQuery->insert('share_external') + ->setValue('share_token', '?') + ->setValue('remote', '?') + ->setValue('name', '?')->setParameter(2, 'irrelevant') + ->setValue('owner', '?')->setParameter(3, 'irrelevant') + ->setValue('user', '?') + ->setValue('mountpoint', '?')->setParameter(5, 'irrelevant') + ->setValue('mountpoint_hash', '?')->setParameter(6, 'irrelevant') + ->setValue('accepted', '?')->setParameter(7, '1'); + for ($i = 0; $i < 21; $i++) { + $shareExternalQuery + ->setParameter(0, "f2c69dad1dc0649f26976fd210fc62e$i") + ->setParameter(1, "https://hostname.tld/owncloud$i") + ->setParameter(4, "user$i"); + $shareExternalQuery->execute(); + } + } + + public function tearDown(): void { + $this->config->deleteAppValue('files_sharing', 'cronjob_scan_external_enabled'); + $this->config->deleteAppValue('files_sharing', 'cronjob_scan_external_min_login'); + $this->config->deleteAppValue('files_sharing', 'cronjob_scan_external_min_scan'); + $this->config->deleteAppValue('files_sharing', 'cronjob_scan_external_batch'); + $this->config->deleteAppValue('files_sharing', 'cronjob_scan_external_offset'); + + $shareExternalQuery = $this->connection->getQueryBuilder(); + $shareExternalQuery->delete('share_external') + ->where($shareExternalQuery->expr()->eq('share_token', $shareExternalQuery->createParameter('share_token'))); + + for ($i = 0; $i < 21; $i++) { + $shareExternalQuery->setParameter('share_token', "f2c69dad1dc0649f26976fd210fc62e$i"); + $shareExternalQuery->execute(); + } + + parent::tearDown(); + } + + public function testFixDI() { + $exceptionThrown = false; + try { + $scanFiles = new ScanExternalSharesJob(); + } catch (\Exception $e) { + $exceptionThrown = true; + } + $this->assertFalse($exceptionThrown); + } + + public function testNotEnabled() { + $scanShares = $this->getScanSharesMockForRun(); + $scanShares->expects($this->never()) + ->method('scan'); + + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_enabled', 'no'); + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_offset', 10); + $this->invokePrivate($scanShares, 'run', [[]]); + } + + public function providesRunHandlesOffset() { + return [ + // when scanned shares, it should go only through max per session and set proper offset to continue + [true, 2, 2, 2], + [true, 9, 9, 9], + // when not scanned any shares, it should go through all external shares + [false, 2, 21, 0], + [false, 9, 21, 0], + [false, 11, 21, 0], + // when scanned shares, and max per session over self::BATCH=10 + // it should go for next 10, but not reach all 21 shares (should reach 20) + [true, 11, 20, 20], + // test for even max per session + [true, 100, 21, 0], + // test for even max per session + [false, 100, 21, 0], + ]; + } + + /** + * @dataProvider providesRunHandlesOffset + */ + public function testRunHandlesOffset($scanShareReturn, $scanShareMaxPerSession, $scanShareExpectedRuns, $expectedOffset) { + $scanShares = $this->getScanSharesMockForRun(); + $scanShares->expects($this->exactly($scanShareExpectedRuns)) + ->method('shouldScan') + ->willReturn($scanShareReturn); + + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_batch', $scanShareMaxPerSession); + + $this->assertEquals(0, $this->config->getAppValue('files_sharing', 'cronjob_scan_external_offset', -1)); + $this->invokePrivate($scanShares, 'run', [[]]); + $this->assertEquals($expectedOffset, $this->config->getAppValue('files_sharing', 'cronjob_scan_external_offset', -1)); + } + + public function testRunContinuesFromOffset() { + $scanShares = $this->getScanSharesMockForRun(); + $scanShares->expects($this->exactly(2)) + ->method('shouldScan') + ->willReturn(true); + + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_batch', 2); + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_offset', 10); + + $this->assertEquals(10, $this->config->getAppValue('files_sharing', 'cronjob_scan_external_offset', -1)); + $this->invokePrivate($scanShares, 'run', [[]]); + $this->assertEquals(12, $this->config->getAppValue('files_sharing', 'cronjob_scan_external_offset', -1)); + } + + public function testRunUpdatesLastTime() { + $scanShares = $this->getScanSharesMockForRun(); + $scanShares->expects($this->exactly(2)) + ->method('shouldScan') + ->willReturn(true); + + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_batch', 2); + $this->config->setAppValue('files_sharing', 'cronjob_scan_external_offset', 0); + + $qb = $this->connection->getQueryBuilder(); + $qb->select('lastscan') + ->from('share_external') + ->where($qb->expr()->eq('remote', $qb->expr()->literal('https://hostname.tld/owncloud1'))); + + $res = $qb->execute()->fetchAll(); + $this->assertNull($res[0]['lastscan']); + + $this->invokePrivate($scanShares, 'run', [[]]); + + $res = $qb->execute()->fetchAll(); + $this->assertNotNull($res[0]['lastscan']); + } + + public function testScanShareNoUser() { + $scanShares = $this->getScanSharesMockFoScan(); + + $this->userManager->expects($this->exactly(1)) + ->method('get') + ->willReturn(null); + + $share = [ + 'share_token' => 'test', + 'user' => 'test', + 'remote' => 'test', + 'token' => 'test', + 'password' => 'test', + 'mountpoint' => 'test', + 'owner' => 'test' + ]; + $lastLoginThreshold = '1'; + $lastScanThreshold = '1'; + $result = $this->invokePrivate($scanShares, 'shouldScan', [$share, $lastLoginThreshold, $lastScanThreshold]); + + $this->assertEquals(false, $result); + } + + public function testScanShareInvalidLastLogin() { + $scanShares = $this->getScanSharesMockFoScan(); + + $user = $this->createMock(IUser::class); + $user->expects($this->exactly(1)) + ->method('getLastLogin') + ->willReturn(\time() - 2); + + $this->userManager->expects($this->exactly(1)) + ->method('get') + ->willReturn($user); + + $share = [ + 'share_token' => 'test', + 'user' => 'test', + 'remote' => 'test', + 'token' => 'test', + 'password' => 'test', + 'mountpoint' => 'test', + 'owner' => 'test' + ]; + $lastLoginThreshold = '1'; + $lastScanThreshold = '1'; + $result = $this->invokePrivate($scanShares, 'shouldScan', [$share, $lastLoginThreshold, $lastScanThreshold]); + + $this->assertEquals(false, $result); + } + + public function testScanShareInvalidLastScan() { + $scanShares = $this->getScanSharesMockFoScan(); + + $user = $this->createMock(IUser::class); + $user->expects($this->exactly(1)) + ->method('getLastLogin') + ->willReturn(\time()); + + $this->userManager->expects($this->exactly(1)) + ->method('get') + ->willReturn($user); + + $share = [ + 'share_token' => 'test', + 'user' => 'test', + 'remote' => 'test', + 'token' => 'test', + 'password' => 'test', + 'mountpoint' => 'test', + 'lastscan' => \time(), + 'owner' => 'test' + ]; + $lastLoginThreshold = '1'; + $lastScanThreshold = '1'; + $result = $this->invokePrivate($scanShares, 'shouldScan', [$share, $lastLoginThreshold, $lastScanThreshold]); + + $this->assertEquals(false, $result); + } + + public function testScanShareNotUpdated() { + $scanShares = $this->getScanSharesMockFoScan(); + + $storage = $this->createMock(Storage::class); + $mount = $this->createMock(Mount::class); + + $this->externalManager->expects($this->exactly(1)) + ->method('getMount') + ->willReturn($mount); + + $mount->expects($this->exactly(1)) + ->method('getStorage') + ->willReturn($storage); + + $mount->expects($this->exactly(1)) + ->method('getStorage') + ->willReturn($storage); + + $storage->expects($this->exactly(1)) + ->method('hasUpdated') + ->willReturn(false); + + $storage->expects($this->never()) + ->method('getScanner'); + + $share = [ + 'share_token' => 'test', + 'user' => 'test', + 'remote' => 'test', + 'token' => 'test', + 'password' => 'test', + 'mountpoint' => 'test', + 'lastscan' => null, + 'owner' => 'test' + ]; + $lastLoginThreshold = 1; + $lastScanThreshold = 1; + $result = $this->invokePrivate($scanShares, 'scan', [$share, $lastLoginThreshold, $lastScanThreshold]); + + $this->assertEquals(false, $result); + } + + public function providesScanShareExceptions() { + $scanner = $this->createMock(Scanner::class); + $propagator = $this->createMock(Propagator::class); + $storage = $this->createMock(Storage::class); + $storage->expects($this->exactly(1)) + ->method('hasUpdated') + ->willReturn(true); + $storage->expects($this->exactly(1)) + ->method('getPropagator') + ->willReturn($propagator); + $storage->expects($this->exactly(1)) + ->method('getScanner') + ->willReturn($scanner); + $scanner->expects($this->exactly(1)) + ->method('scan') + ->willReturn(true); + $tests[] = [$storage]; + + $scanner = $this->createMock(Scanner::class); + $propagator = $this->createMock(Propagator::class); + $storage = $this->createMock(Storage::class); + $storage->expects($this->exactly(1)) + ->method('hasUpdated') + ->willReturn(true); + $storage->expects($this->exactly(1)) + ->method('getPropagator') + ->willReturn($propagator); + $storage->expects($this->exactly(1)) + ->method('getScanner') + ->willReturn($scanner); + $scanner->expects($this->exactly(1)) + ->method('scan') + ->willReturn(true); + $scanner->method('scan')->willThrowException(new \Exception()); + $tests[] = [$storage]; + + $scanner = $this->createMock(Scanner::class); + $propagator = $this->createMock(Propagator::class); + $storage = $this->createMock(Storage::class); + $storage->expects($this->exactly(1)) + ->method('hasUpdated') + ->willReturn(true); + $storage->expects($this->exactly(1)) + ->method('getPropagator') + ->willReturn($propagator); + $storage->expects($this->exactly(1)) + ->method('getScanner') + ->willReturn($scanner); + $scanner->expects($this->exactly(1)) + ->method('scan') + ->willReturn(true); + $scanner->method('scan')->willThrowException(new NoUserException()); + $tests[] = [$storage]; + + $scanner = $this->createMock(Scanner::class); + $propagator = $this->createMock(Propagator::class); + $storage = $this->createMock(Storage::class); + $storage->expects($this->exactly(1)) + ->method('hasUpdated') + ->willReturn(true); + $storage->expects($this->exactly(1)) + ->method('getPropagator') + ->willReturn($propagator); + $storage->expects($this->exactly(1)) + ->method('getScanner') + ->willReturn($scanner); + $scanner->expects($this->exactly(1)) + ->method('scan') + ->willReturn(true); + $scanner->method('scan')->willThrowException(new StorageNotAvailableException()); + $tests[] = [$storage]; + return $tests; + } + + /** + * @dataProvider providesScanShareExceptions + */ + public function testScanShareExceptions($storage) { + $scanShares = $this->getScanSharesMockFoScan(); + + $mount = $this->createMock(Mount::class); + + $this->externalManager->expects($this->exactly(1)) + ->method('getMount') + ->willReturn($mount); + + $mount->expects($this->exactly(1)) + ->method('getStorage') + ->willReturn($storage); + + $share = [ + 'share_token' => 'test', + 'user' => 'test', + 'remote' => 'test', + 'token' => 'test', + 'password' => 'test', + 'mountpoint' => 'test', + 'lastscan' => null, + 'owner' => 'test' + ]; + $lastLoginThreshold = 1; + $lastScanThreshold = 1; + $result = $this->invokePrivate($scanShares, 'scan', [$share, $lastLoginThreshold, $lastScanThreshold]); + + $this->assertEquals(true, $result); + } + + private function getScanSharesMockFoScan() { + return $this->getMockBuilder(ScanExternalSharesJob::class) + ->setConstructorArgs([ + $this->connection, + $this->config, + $this->userManager, + $this->createMock(ILogger::class), + $this->externalManager, + ]) + ->setMethods([]) + ->getMock(); + } + + private function getScanSharesMockForRun() { + return $this->getMockBuilder(ScanExternalSharesJob::class) + ->setConstructorArgs([ + $this->connection, + $this->config, + $this->userManager, + $this->createMock(ILogger::class), + $this->externalManager, + ]) + ->setMethods(['shouldScan', 'scan']) + ->getMock(); + } +} diff --git a/changelog/unreleased/37391 b/changelog/unreleased/37391 new file mode 100644 index 000000000000..850b86164f33 --- /dev/null +++ b/changelog/unreleased/37391 @@ -0,0 +1,13 @@ +Change: Added federated shares scan cronjob depreciating incoming-shares:poll + +We've fixed the behavior for federated shares poll command that in certain conditions +was producing stale filecache entries, and replaced it by fed shares scan cronjob. + +ScanExternalShares that was added is a background job used to scan external shares +(federated shares) that are eligible for scanning to ensure integrity of the +file cache - i.e. satisfy preconditions as last user login, +last scan and whether root storage updated. + +https://github.com/owncloud/core/pull/37391 +https://github.com/owncloud/enterprise/issues/3902 +https://doc.owncloud.com/server/admin_manual/configuration/files/federated_cloud_sharing_configuration.html \ No newline at end of file diff --git a/lib/public/Files/Cache/IScanner.php b/lib/public/Files/Cache/IScanner.php index ebcefc5dbe67..8d76fe0107ed 100644 --- a/lib/public/Files/Cache/IScanner.php +++ b/lib/public/Files/Cache/IScanner.php @@ -31,6 +31,7 @@ interface IScanner { const SCAN_RECURSIVE = true; const SCAN_SHALLOW = false; + const REUSE_NONE = 0; const REUSE_ETAG = 1; const REUSE_SIZE = 2; const REUSE_ONLY_FOR_FILES = 4; // apply the reuse (either tag or size) only to files, not folders