Add fed shares scanner batched cronjob

apps/federatedfilesharing/lib/Command/PollIncomingShares.php

@@ -84,6 +84,7 @@ protected function configure() {
 	 * @return int|null|void
 	 */
 	public function execute(InputInterface $input, OutputInterface $output) {
+		$output->writeln("WARNING: incoming-shares:poll has been depreciated and replaced by periodic external shares cronjob. Please check Federated Cloud Sharing settings and documentation.");
 		if ($this->externalMountProvider === null) {
 			$output->writeln("Polling is not possible when files_sharing app is disabled. Please enable it with 'occ app:enable files_sharing'");
 			return 1;

apps/federatedfilesharing/lib/FederatedShareProvider.php

@@ -959,6 +959,16 @@ public function userDeletedFromGroup($uid, $gid) {
 		return;
 	}
 
+	/**
+	 * check if scan of federated shares from other ownCloud instances should be performed
+	 *
+	 * @return bool
+	 */
+	public function isCronjobScanExternalEnabled() {
+		$result = $this->config->getAppValue('files_sharing', 'cronjob_scan_external_enabled', 'no');
+		return ($result === 'yes') ? true : false;
+	}
+
 	/**
 	 * check if users from other ownCloud instances are allowed to mount public links share by this instance
 	 *

apps/federatedfilesharing/lib/Panels/AdminPanel.php

@@ -53,6 +53,7 @@ public function getSectionID() {
 
 	public function getPanel() {
 		$tmpl = new Template('federatedfilesharing', 'settings-admin');
+		$tmpl->assign('cronjobScanExternalEnabled', $this->shareProvider->isCronjobScanExternalEnabled());
 		$tmpl->assign('outgoingServer2serverShareEnabled', $this->shareProvider->isOutgoingServer2serverShareEnabled());
 		$tmpl->assign('incomingServer2serverShareEnabled', $this->shareProvider->isIncomingServer2serverShareEnabled());
 		$tmpl->assign(

apps/federatedfilesharing/templates/settings-admin.php

@@ -10,6 +10,16 @@
 		title="<?php p($l->t('Open documentation'));?>"
 		href="<?php p(link_to_docs('admin-sharing-federated')); ?>"></a>
 
+	<p>
+		<input type="checkbox" name="cronjob_scan_external_enabled" id="cronjobScanExternalEnabled" class="checkbox"
+			   value="1" <?php if ($_['cronjobScanExternalEnabled']) {
+	print_unescaped('checked="checked"');
+		} ?> />
+		<label for="cronjobScanExternalEnabled">
+			<?php p($l->t('Periodically synchronize outdated federated shares for active users'));?>
+		</label>
+	</p>
+
 	<p>
 		<input type="checkbox" name="outgoing_server2server_share_enabled" id="outgoingServer2serverShareEnabled" class="checkbox"
 			   value="1" <?php if ($_['outgoingServer2serverShareEnabled']) {

apps/federatedfilesharing/tests/FederatedShareProviderTest.php

@@ -857,6 +857,22 @@ public function testDeleteUser($owner, $initiator, $recipient, $deletedUser, $ro
 		$this->assertCount($rowDeleted ? 0 : 1, $data);
 	}
 
+	/**
+	 * @dataProvider dataTestFederatedSharingSettings
+	 *
+	 * @param string $isEnabled
+	 * @param bool $expected
+	 */
+	public function testIsCronjobScanExternalEnabled($isEnabled, $expected) {
+		$this->config->expects($this->once())->method('getAppValue')
+			->with('files_sharing', 'cronjob_scan_external_enabled', 'no')
+			->willReturn($isEnabled);
+
+		$this->assertSame($expected,
+			$this->provider->isCronjobScanExternalEnabled()
+		);
+	}
+
 	/**
 	 * @dataProvider dataTestFederatedSharingSettings
 	 *

apps/federatedfilesharing/tests/Panels/AdminPanelTest.php

@@ -55,6 +55,7 @@ public function testGetPriority() {
 	}
 
 	public function testGetPanel() {
+		$this->shareProvider->expects($this->once())->method('isOutgoingServer2serverShareEnabled')->willReturn(false);
 		$this->shareProvider->expects($this->once())->method('isOutgoingServer2serverShareEnabled')->willReturn(true);
 		$this->shareProvider->expects($this->once())->method('isIncomingServer2serverShareEnabled')->willReturn(true);
 		$templateHtml = $this->panel->getPanel()->fetchPage();

apps/files_sharing/appinfo/Migrations/Version20200504211654.php

@@ -0,0 +1,31 @@
+<?php
+namespace OCA\Files_Sharing\Migrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\DBAL\Types\Type;
+use OCP\Migration\ISchemaMigration;
+
+/**
+ * Auto-generated migration step: Please modify to your needs!
+ */
+class Version20200504211654 implements ISchemaMigration {
+	public function changeSchema(Schema $schema, array $options) {
+		$prefix = $options['tablePrefix'];
+
+		if ($schema->hasTable("${prefix}share_external")) {
+			$table = $schema->getTable("${prefix}share_external");
+
+			if (!$table->hasColumn('lastscan')) {
+				$table->addColumn(
+					'lastscan',
+					Type::BIGINT,
+					[
+						'length' => 11,
+						'unsigned' => true,
+						'notnull' => false,
+					]
+				);
+			}
+		}
+	}
+}

apps/files_sharing/appinfo/info.xml

@@ -10,7 +10,7 @@ Turning the feature off removes shared files and folders on the server for all s
 	<licence>AGPL</licence>
 	<author>Michael Gapczynski, Bjoern Schiessle</author>
 	<default_enable/>
-	<version>0.12.0</version>
+	<version>0.13.0</version>
 	<types>
 		<filesystem/>
 	</types>
@@ -35,6 +35,7 @@ Turning the feature off removes shared files and folders on the server for all s
 	<background-jobs>
 		<job>OCA\Files_Sharing\DeleteOrphanedSharesJob</job>
 		<job>OCA\Files_Sharing\ExpireSharesJob</job>
+		<job>OCA\Files_Sharing\External\ScanExternalSharesJob</job>
 	</background-jobs>
 
 	<commands>

apps/files_sharing/lib/External/ScanExternalSharesJob.php

@@ -0,0 +1,253 @@
+<?php
+/**
+ * @author Piotr Mrowczynski <piotr@owncloud.com>
+ *
+ * @copyright Copyright (c) 2020, ownCloud GmbH
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OCA\Files_Sharing\External;
+
+use OCP\Files\Cache\IScanner;
+use OCP\IConfig;
+use OCP\IDBConnection;
+use OCP\ILogger;
+use OCP\IUserManager;
+use OC\User\NoUserException;
+use OCP\Files\StorageNotAvailableException;
+
+
+/**
+ * Class ScanExternalShares is a background job used to run the external shares
+ * scanner over the user accounts to ensure integrity of the file cache.
+ *
+ * @package OCA\Files_Sharing\External\BackgroundJob
+ */
+class ScanExternalSharesJob extends \OC\BackgroundJob\TimedJob {
+	/** @var IDBConnection */
+	private $connection;
+	/** @var Manager */
+	private $externalManager;
+	/** @var IConfig */
+	private $config;
+	/** @var IUserManager */
+	private $userManager;
+	/** @var ILogger */
+	private $logger;
+
+	const DEFAULT_MIN_LAST_SCAN = 1*60*60;
+	const DEFAULT_MIN_LOGIN = 24*60*60;
+	const DEFAULT_SHARES_PER_SESSION = 100;
+	const BATCH_SIZE = 10;
+
+	public function __construct(IDBConnection $connection = null,
+								IConfig $config = null,
+								IUserManager $userManager = null,
+								ILogger $logger = null,
+								Manager $externalManager = null) {
+		// Run once per 10 minutes
+		$this->setInterval(60 * 10);
+
+		if ($logger === null || $userManager === null || $config === null || $connection === null || $externalManager === null) {
+			$this->fixDIForJobs();
+		} else {
+			$this->connection = $connection;
+			$this->externalManager = $externalManager;
+			$this->config = $config;
+			$this->userManager = $userManager;
+			$this->logger = $logger;
+		}
+	}
+
+	protected function fixDIForJobs() {
+		$this->connection = \OC::$server->getDatabaseConnection();
+		$this->config = \OC::$server->getConfig();
+		$this->userManager = \OC::$server->getUserManager();
+		$this->logger = \OC::$server->getLogger();
+		$this->externalManager = new Manager(
+			$this->connection,
+			\OC\Files\Filesystem::getMountManager(),
+			\OC\Files\Filesystem::getLoader(),
+			\OC::$server->getNotificationManager(),
+			\OC::$server->getEventDispatcher(),
+			null
+		);
+	}
+
+	/**
+	 * @param $argument
+	 * @throws \Exception
+	 */
+	protected function run($argument) {
+		$enabled = $this->config->getAppValue('files_sharing', 'cronjob_scan_external_enabled', 'no');
+		if ($enabled === 'yes') {
+			return;
+		}
+
+		$lastLoginThreshold = $this->config->getAppValue('files_sharing', 'cronjob_scan_external_min_login', self::DEFAULT_MIN_LOGIN);
+		$lastScanThreshold = $this->config->getAppValue('files_sharing', 'cronjob_scan_external_min_scan', self::DEFAULT_MIN_LAST_SCAN);
+		$maxSharesPerSession = $this->config->getAppValue('files_sharing', 'cronjob_scan_external_batch', self::DEFAULT_SHARES_PER_SESSION);
+		$batchPerSession = \min($maxSharesPerSession, self::BATCH_SIZE);
+
+		$scannedShares = 0;
+
+		do {
+			$offset = $this->config->getAppValue('files_sharing', 'cronjob_scan_external_offset', 0);
+			$shares = $this->getAcceptedShares($batchPerSession, $offset);
+
+			$searchedBatch = 0;
+			foreach ($shares as $share) {
+				$scanned = $this->scanShare($share, $lastLoginThreshold, $lastScanThreshold);
+				if ($scanned) {
+					$this->updateLastScanned($share['id'], \time());
+					$scannedShares += 1;
+				}
+				$searchedBatch += 1;
+			}
+
+			$this->logger->debug(
+				"Fed share scanner performed scan of $scannedShares/$searchedBatch shares at offset $offset"
+			);
+
+			if (count($shares) < $batchPerSession) {
+				// next run wont have any users to scan,
+				// as we returned less than the limit
+				$offset = 0;
+			} else {
+				$offset += $batchPerSession;
+			}
+
+			$this->config->setAppValue('files_sharing', 'cronjob_scan_external_offset', $offset);
+		} while ($offset !== 0 && $scannedShares < $maxSharesPerSession);
+	}
+
+	protected function scanShare($share, $lastLoginThreshold, $lastScanThreshold) {
+		$now = \time();
+
+		// check last login
+		$user = $this->userManager->get($share['user']);
+		if ($user === null) {
+			$this->logger->debug(
+				"Will not scan fed share {$share['mountpoint']} for uid {$share['user']}, user does not exist"
+			);
+			return false;
+		}
+		$lastLogin = $user->getLastLogin();
+		if ($lastLogin + $lastLoginThreshold < $now) {
+			$this->logger->debug(
+				"Will not scan fed share {$share['mountpoint']} for uid {$share['user']}, user did not login in last $lastLoginThreshold seconds"
+			);
+			return false;
+		}
+
+		// check last scan
+		$lastScan = $share['lastscan'] ? \intval($share['lastscan']) : 0;
+		if ($lastScan + $lastScanThreshold > $now) {
+			$this->logger->debug(
+				"Will not scan fed share {$share['mountpoint']} for uid {$share['user']}, share has been already scanned in last $lastScanThreshold seconds"
+			);
+			return false;
+		}
+
+		// get mount
+		$options = [
+			'remote'	=> $share['remote'],
+			'token'		=> $share['share_token'],
+			'password'	=> $share['password'],
+			'mountpoint'	=> $share['mountpoint'],
+			'owner'		=> $share['owner']
+		];
+		$this->externalManager->setUid($share['user']);
+		$mount = $this->externalManager->getMount($options);
+		$this->externalManager->setUid(null);
+
+		// check if root storage updated
+		$storage = $mount->getStorage();
+		$localMtime = $storage->filemtime('');
+		$hasUpdated = $storage->hasUpdated('', $localMtime);
+		if (!$hasUpdated) {
+			$this->logger->debug(
+				"Scanned fed share {$share['mountpoint']} for uid {$share['user']}, share is up to date"
+			);
+			return true;
+		}
+
+		// scan recursive, and do not reuse anything
+		// as we need to force scanning of the external share storage
+		try {
+			$propagator = $storage->getPropagator();
+			$propagator->beginBatch();
+			$storage->getScanner()->scan('', IScanner::SCAN_RECURSIVE, IScanner::REUSE_NONE);
+			$propagator->commitBatch();
+
+			$this->logger->debug(
+				"Scanned fed share {$share['mountpoint']} for uid {$share['user']} with last login $lastLogin and last scan {$share['lastscan']}"
+			);
+		} catch (NoUserException $e) {
+			// uid was null so we need to set it
+			$this->externalManager->setUid($share['user']);
+			$this->externalManager->removeShare($mount->getMountPoint());
+			// and now we need to reset uid back to null
+			$this->externalManager->setUid(null);
+			$this->logger->debug(
+				"Remote {$share['remote']} reports that external share with {$share['mountpoint']} for uid {$share['user']} no longer exists. Removing it.."
+			);
+		} catch (StorageNotAvailableException $e) {
+			$reason = $e->getMessage();
+			$this->logger->debug(
+				"Skipping external share {$share['mountpoint']} for uid {$share['user']} from remote {$share['remote']} as share is unreachable. Reason: {$reason}"
+			);
+		} catch (\Exception $e) {
+			$this->logger->debug(
+				"Skipping external share {$share['mountpoint']} for uid {$share['user']} from remote {$share['remote']}  due to internal server error"
+			);
+			$this->logger->logException($e, ['app' => 'federatedfilesharing']);
+		}
+
+		$this->tearDownStorage();
+
+		return true;
+	}
+
+	protected function tearDownStorage() {
+		\OC_Util::tearDownFS();
+	}
+
+	protected function getAcceptedShares($limit, $offset) {
+		$qb = $this->connection->getQueryBuilder();
+		$qb->select('id', 'remote', 'share_token', 'password', 'mountpoint', 'owner', 'user', 'mountpoint', 'lastscan')
+			->from('share_external')
+			->where($qb->expr()->eq('accepted', $qb->expr()->literal('1')));
+
+		$qb->setMaxResults($limit);
+		$qb->setFirstResult($offset);
+		$qb->orderBy('id');
+
+		$cursor = $qb->execute();
+		return $cursor->fetchAll();
+	}
+
+
+	protected function updateLastScanned($id, $updatedTime) {
+		$qb = $this->connection->getQueryBuilder();
+		$qb->update('share_external')
+			->set('lastscan', $qb->createNamedParameter($updatedTime))
+			->where($qb->expr()->eq('id', $qb->createNamedParameter($id)));
+
+		$result = $qb->execute();
+		return $result === 1;
+	}
+}