Skip to content

Commit

Permalink
Merge pull request #18351 from nextcloud/enhancement/typed-filesystem…
Browse files Browse the repository at this point in the history
…-events

Add typed events for the filesystem/scanner
  • Loading branch information
blizzz authored Dec 13, 2019
2 parents 6426067 + b81b824 commit ae6460c
Show file tree
Hide file tree
Showing 15 changed files with 511 additions and 52 deletions.
33 changes: 13 additions & 20 deletions apps/files/lib/BackgroundJob/ScanFiles.php
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
namespace OCA\Files\BackgroundJob;

use OC\Files\Utils\Scanner;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\ILogger;
use OCP\IUser;
use OCP\IUserManager;
Expand All @@ -41,39 +41,31 @@ class ScanFiles extends \OC\BackgroundJob\TimedJob {
private $config;
/** @var IUserManager */
private $userManager;
/** @var IDBConnection */
private $dbConnection;
/** @var IEventDispatcher */
private $dispatcher;
/** @var ILogger */
private $logger;

/** Amount of users that should get scanned per execution */
const USERS_PER_SESSION = 500;

/**
* @param IConfig|null $config
* @param IUserManager|null $userManager
* @param IDBConnection|null $dbConnection
* @param IEventDispatcher|null $dispatcher
* @param ILogger|null $logger
*/
public function __construct(IConfig $config = null,
IUserManager $userManager = null,
IDBConnection $dbConnection = null,
IEventDispatcher $dispatcher = null,
ILogger $logger = null) {
// Run once per 10 minutes
$this->setInterval(60 * 10);

if (is_null($userManager) || is_null($config)) {
$this->fixDIForJobs();
} else {
$this->config = $config;
$this->userManager = $userManager;
$this->logger = $logger;
}
}

protected function fixDIForJobs() {
$this->config = \OC::$server->getConfig();
$this->userManager = \OC::$server->getUserManager();
$this->logger = \OC::$server->getLogger();
$this->config = $config ?? \OC::$server->getConfig();
$this->userManager = $userManager ?? \OC::$server->getUserManager();
$this->dispatcher = $dispatcher ?? \OC::$server->query(IEventDispatcher::class);
$this->logger = $logger ?? \OC::$server->getLogger();
}

/**
Expand All @@ -83,7 +75,8 @@ protected function runScanner(IUser $user) {
try {
$scanner = new Scanner(
$user->getUID(),
$this->dbConnection,
null,
$this->dispatcher,
$this->logger
);
$scanner->backgroundScan('');
Expand All @@ -101,7 +94,7 @@ protected function run($argument) {
if ($this->config->getSystemValueBool('files_no_background_scan', false)) {
return;
}

$offset = $this->config->getAppValue('files', 'cronjob_scan_files', 0);
$users = $this->userManager->search('', self::USERS_PER_SESSION, $offset);
if (!count($users)) {
Expand Down
3 changes: 2 additions & 1 deletion apps/files/lib/Command/Scan.php
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
use OC\Core\Command\Base;
use OC\Core\Command\InterruptedException;
use OC\ForbiddenException;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\Mount\IMountPoint;
use OCP\Files\NotFoundException;
use OCP\Files\StorageNotAvailableException;
Expand Down Expand Up @@ -114,7 +115,7 @@ public function checkScanWarning($fullPath, OutputInterface $output) {

protected function scanFiles($user, $path, OutputInterface $output, $backgroundScan = false, $recursive = true, $homeOnly = false) {
$connection = $this->reconnectToDatabase($output);
$scanner = new \OC\Files\Utils\Scanner($user, $connection, \OC::$server->getLogger());
$scanner = new \OC\Files\Utils\Scanner($user, $connection, \OC::$server->query(IEventDispatcher::class), \OC::$server->getLogger());

# check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception

Expand Down
3 changes: 2 additions & 1 deletion apps/files/lib/Command/ScanAppData.php
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
use OC\Core\Command\Base;
use OC\Core\Command\InterruptedException;
use OC\ForbiddenException;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
use OCP\Files\StorageNotAvailableException;
Expand Down Expand Up @@ -85,7 +86,7 @@ protected function scanFiles(OutputInterface $output) {
}

$connection = $this->reconnectToDatabase($output);
$scanner = new \OC\Files\Utils\Scanner(null, $connection, \OC::$server->getLogger());
$scanner = new \OC\Files\Utils\Scanner(null, $connection, \OC::$server->query(IEventDispatcher::class), \OC::$server->getLogger());

# check on each file/folder if there was a user interrupt (ctrl-c) and throw an exception
$scanner->listen('\OC\Files\Utils\Scanner', 'scanFile', function ($path) use ($output) {
Expand Down
7 changes: 7 additions & 0 deletions lib/composer/composer/autoload_classmap.php
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,13 @@
'OCP\\Files\\Config\\IUserMountCache' => $baseDir . '/lib/public/Files/Config/IUserMountCache.php',
'OCP\\Files\\EmptyFileNameException' => $baseDir . '/lib/public/Files/EmptyFileNameException.php',
'OCP\\Files\\EntityTooLargeException' => $baseDir . '/lib/public/Files/EntityTooLargeException.php',
'OCP\\Files\\Events\\BeforeFileScannedEvent' => $baseDir . '/lib/public/Files/Events/BeforeFileScannedEvent.php',
'OCP\\Files\\Events\\BeforeFolderScannedEvent' => $baseDir . '/lib/public/Files/Events/BeforeFolderScannedEvent.php',
'OCP\\Files\\Events\\FileCacheUpdated' => $baseDir . '/lib/public/Files/Events/FileCacheUpdated.php',
'OCP\\Files\\Events\\FileScannedEvent' => $baseDir . '/lib/public/Files/Events/FileScannedEvent.php',
'OCP\\Files\\Events\\FolderScannedEvent' => $baseDir . '/lib/public/Files/Events/FolderScannedEvent.php',
'OCP\\Files\\Events\\NodeAddedToCache' => $baseDir . '/lib/public/Files/Events/NodeAddedToCache.php',
'OCP\\Files\\Events\\NodeRemovedFromCache' => $baseDir . '/lib/public/Files/Events/NodeRemovedFromCache.php',
'OCP\\Files\\File' => $baseDir . '/lib/public/Files/File.php',
'OCP\\Files\\FileInfo' => $baseDir . '/lib/public/Files/FileInfo.php',
'OCP\\Files\\FileNameTooLongException' => $baseDir . '/lib/public/Files/FileNameTooLongException.php',
Expand Down
7 changes: 7 additions & 0 deletions lib/composer/composer/autoload_static.php
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,13 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OCP\\Files\\Config\\IUserMountCache' => __DIR__ . '/../../..' . '/lib/public/Files/Config/IUserMountCache.php',
'OCP\\Files\\EmptyFileNameException' => __DIR__ . '/../../..' . '/lib/public/Files/EmptyFileNameException.php',
'OCP\\Files\\EntityTooLargeException' => __DIR__ . '/../../..' . '/lib/public/Files/EntityTooLargeException.php',
'OCP\\Files\\Events\\BeforeFileScannedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Events/BeforeFileScannedEvent.php',
'OCP\\Files\\Events\\BeforeFolderScannedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Events/BeforeFolderScannedEvent.php',
'OCP\\Files\\Events\\FileCacheUpdated' => __DIR__ . '/../../..' . '/lib/public/Files/Events/FileCacheUpdated.php',
'OCP\\Files\\Events\\FileScannedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Events/FileScannedEvent.php',
'OCP\\Files\\Events\\FolderScannedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Events/FolderScannedEvent.php',
'OCP\\Files\\Events\\NodeAddedToCache' => __DIR__ . '/../../..' . '/lib/public/Files/Events/NodeAddedToCache.php',
'OCP\\Files\\Events\\NodeRemovedFromCache' => __DIR__ . '/../../..' . '/lib/public/Files/Events/NodeRemovedFromCache.php',
'OCP\\Files\\File' => __DIR__ . '/../../..' . '/lib/public/Files/File.php',
'OCP\\Files\\FileInfo' => __DIR__ . '/../../..' . '/lib/public/Files/FileInfo.php',
'OCP\\Files\\FileNameTooLongException' => __DIR__ . '/../../..' . '/lib/public/Files/FileNameTooLongException.php',
Expand Down
56 changes: 35 additions & 21 deletions lib/private/Files/Utils/Scanner.php
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,18 @@
use OC\Hooks\PublicEmitter;
use OC\Lock\DBLockingProvider;
use OCA\Files_Sharing\SharedStorage;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\Events\BeforeFileScannedEvent;
use OCP\Files\Events\BeforeFolderScannedEvent;
use OCP\Files\Events\NodeAddedToCache;
use OCP\Files\Events\FileCacheUpdated;
use OCP\Files\Events\NodeRemovedFromCache;
use OCP\Files\Events\FileScannedEvent;
use OCP\Files\Events\FolderScannedEvent;
use OCP\Files\NotFoundException;
use OCP\Files\Storage\IStorage;
use OCP\Files\StorageNotAvailableException;
use OCP\IDBConnection;
use OCP\ILogger;

/**
Expand All @@ -53,19 +62,16 @@
class Scanner extends PublicEmitter {
const MAX_ENTRIES_TO_COMMIT = 10000;

/**
* @var string $user
*/
/** @var string $user */
private $user;

/**
* @var \OCP\IDBConnection
*/
/** @var IDBConnection */
protected $db;

/**
* @var ILogger
*/
/** @var IEventDispatcher */
private $dispatcher;

/** @var ILogger */
protected $logger;

/**
Expand All @@ -84,13 +90,15 @@ class Scanner extends PublicEmitter {

/**
* @param string $user
* @param \OCP\IDBConnection $db
* @param IDBConnection|null $db
* @param IEventDispatcher $dispatcher
* @param ILogger $logger
*/
public function __construct($user, $db, ILogger $logger) {
$this->logger = $logger;
public function __construct($user, $db, IEventDispatcher $dispatcher, ILogger $logger) {
$this->user = $user;
$this->db = $db;
$this->dispatcher = $dispatcher;
$this->logger = $logger;
// when DB locking is used, no DB transactions will be used
$this->useTransaction = !(\OC::$server->getLockingProvider() instanceof DBLockingProvider);
}
Expand Down Expand Up @@ -121,18 +129,21 @@ protected function getMounts($dir) {
*/
protected function attachListener($mount) {
$scanner = $mount->getStorage()->getScanner();
$emitter = $this;
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function ($path) use ($mount, $emitter) {
$emitter->emit('\OC\Files\Utils\Scanner', 'scanFile', array($mount->getMountPoint() . $path));
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function ($path) use ($mount) {
$this->emit('\OC\Files\Utils\Scanner', 'scanFile', array($mount->getMountPoint() . $path));
$this->dispatcher->dispatchTyped(new BeforeFileScannedEvent($mount->getMountPoint() . $path));
});
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function ($path) use ($mount, $emitter) {
$emitter->emit('\OC\Files\Utils\Scanner', 'scanFolder', array($mount->getMountPoint() . $path));
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function ($path) use ($mount) {
$this->emit('\OC\Files\Utils\Scanner', 'scanFolder', array($mount->getMountPoint() . $path));
$this->dispatcher->dispatchTyped(new BeforeFolderScannedEvent($mount->getMountPoint() . $path));
});
$scanner->listen('\OC\Files\Cache\Scanner', 'postScanFile', function ($path) use ($mount, $emitter) {
$emitter->emit('\OC\Files\Utils\Scanner', 'postScanFile', array($mount->getMountPoint() . $path));
$scanner->listen('\OC\Files\Cache\Scanner', 'postScanFile', function ($path) use ($mount) {
$this->emit('\OC\Files\Utils\Scanner', 'postScanFile', array($mount->getMountPoint() . $path));
$this->dispatcher->dispatchTyped(new FileScannedEvent($mount->getMountPoint() . $path));
});
$scanner->listen('\OC\Files\Cache\Scanner', 'postScanFolder', function ($path) use ($mount, $emitter) {
$emitter->emit('\OC\Files\Utils\Scanner', 'postScanFolder', array($mount->getMountPoint() . $path));
$scanner->listen('\OC\Files\Cache\Scanner', 'postScanFolder', function ($path) use ($mount) {
$this->emit('\OC\Files\Utils\Scanner', 'postScanFolder', array($mount->getMountPoint() . $path));
$this->dispatcher->dispatchTyped(new FolderScannedEvent($mount->getMountPoint() . $path));
});
}

Expand Down Expand Up @@ -225,12 +236,15 @@ public function scan($dir = '', $recursive = \OC\Files\Cache\Scanner::SCAN_RECUR

$scanner->listen('\OC\Files\Cache\Scanner', 'removeFromCache', function ($path) use ($storage) {
$this->postProcessEntry($storage, $path);
$this->dispatcher->dispatchTyped(new NodeRemovedFromCache($storage, $path));
});
$scanner->listen('\OC\Files\Cache\Scanner', 'updateCache', function ($path) use ($storage) {
$this->postProcessEntry($storage, $path);
$this->dispatcher->dispatchTyped(new FileCacheUpdated($storage, $path));
});
$scanner->listen('\OC\Files\Cache\Scanner', 'addToCache', function ($path) use ($storage) {
$this->postProcessEntry($storage, $path);
$this->dispatcher->dispatchTyped(new NodeAddedToCache($storage, $path));
});

if (!$storage->file_exists($relativePath)) {
Expand Down
56 changes: 56 additions & 0 deletions lib/public/Files/Events/BeforeFileScannedEvent.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
<?php

declare(strict_types=1);

/**
* @copyright 2019 Christoph Wurst <christoph@winzerhof-wurst.at>
*
* @author 2019 Christoph Wurst <christoph@winzerhof-wurst.at>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

namespace OCP\Files\Events;

use OCP\EventDispatcher\Event;

/**
* @since 18.0.0
*/
class BeforeFileScannedEvent extends Event {

/** @var string */
private $absolutePath;

/**
* @param string $absolutePath
*
* @since 18.0.0
*/
public function __construct(string $absolutePath) {
parent::__construct();
$this->absolutePath = $absolutePath;
}

/**
* @return string
* @since 18.0.0
*/
public function getAbsolutePath(): string {
return $this->absolutePath;
}

}
56 changes: 56 additions & 0 deletions lib/public/Files/Events/BeforeFolderScannedEvent.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
<?php

declare(strict_types=1);

/**
* @copyright 2019 Christoph Wurst <christoph@winzerhof-wurst.at>
*
* @author 2019 Christoph Wurst <christoph@winzerhof-wurst.at>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

namespace OCP\Files\Events;

use OCP\EventDispatcher\Event;

/**
* @since 18.0.0
*/
class BeforeFolderScannedEvent extends Event {

/** @var string */
private $absolutePath;

/**
* @param string $absolutePath
*
* @since 18.0.0
*/
public function __construct(string $absolutePath) {
parent::__construct();
$this->absolutePath = $absolutePath;
}

/**
* @return string
* @since 18.0.0
*/
public function getAbsolutePath(): string {
return $this->absolutePath;
}

}
Loading

0 comments on commit ae6460c

Please sign in to comment.