Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 4208014

Browse files
committedMar 6, 2024·
add command to scan external storages directly
the main use case of this over simply scanning through is the ability to provide a username and/or password for cases where login credentials are used Signed-off-by: Robin Appelman <robin@icewind.nl>
1 parent 28c3e40 commit 4208014

File tree

6 files changed

+296
-74
lines changed

6 files changed

+296
-74
lines changed
 

‎apps/files_external/appinfo/info.xml

+1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ External storage can be configured using the GUI or at the command line. This se
4747
<command>OCA\Files_External\Command\Backends</command>
4848
<command>OCA\Files_External\Command\Verify</command>
4949
<command>OCA\Files_External\Command\Notify</command>
50+
<command>OCA\Files_External\Command\Scan</command>
5051
</commands>
5152

5253
<settings>

‎apps/files_external/composer/composer/autoload_classmap.php

+2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
'OCA\\Files_External\\Command\\ListCommand' => $baseDir . '/../lib/Command/ListCommand.php',
2020
'OCA\\Files_External\\Command\\Notify' => $baseDir . '/../lib/Command/Notify.php',
2121
'OCA\\Files_External\\Command\\Option' => $baseDir . '/../lib/Command/Option.php',
22+
'OCA\\Files_External\\Command\\Scan' => $baseDir . '/../lib/Command/Scan.php',
23+
'OCA\\Files_External\\Command\\StorageAuthBase' => $baseDir . '/../lib/Command/StorageAuthBase.php',
2224
'OCA\\Files_External\\Command\\Verify' => $baseDir . '/../lib/Command/Verify.php',
2325
'OCA\\Files_External\\Config\\ConfigAdapter' => $baseDir . '/../lib/Config/ConfigAdapter.php',
2426
'OCA\\Files_External\\Config\\ExternalMountPoint' => $baseDir . '/../lib/Config/ExternalMountPoint.php',

‎apps/files_external/composer/composer/autoload_static.php

+2
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ class ComposerStaticInitFiles_External
3434
'OCA\\Files_External\\Command\\ListCommand' => __DIR__ . '/..' . '/../lib/Command/ListCommand.php',
3535
'OCA\\Files_External\\Command\\Notify' => __DIR__ . '/..' . '/../lib/Command/Notify.php',
3636
'OCA\\Files_External\\Command\\Option' => __DIR__ . '/..' . '/../lib/Command/Option.php',
37+
'OCA\\Files_External\\Command\\Scan' => __DIR__ . '/..' . '/../lib/Command/Scan.php',
38+
'OCA\\Files_External\\Command\\StorageAuthBase' => __DIR__ . '/..' . '/../lib/Command/StorageAuthBase.php',
3739
'OCA\\Files_External\\Command\\Verify' => __DIR__ . '/..' . '/../lib/Command/Verify.php',
3840
'OCA\\Files_External\\Config\\ConfigAdapter' => __DIR__ . '/..' . '/../lib/Config/ConfigAdapter.php',
3941
'OCA\\Files_External\\Config\\ExternalMountPoint' => __DIR__ . '/..' . '/../lib/Config/ExternalMountPoint.php',

‎apps/files_external/lib/Command/Notify.php

+6-74
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,13 @@
3030
namespace OCA\Files_External\Command;
3131

3232
use Doctrine\DBAL\Exception\DriverException;
33-
use OC\Core\Command\Base;
34-
use OCA\Files_External\Lib\InsufficientDataForMeaningfulAnswerException;
35-
use OCA\Files_External\Lib\StorageConfig;
3633
use OCA\Files_External\Service\GlobalStoragesService;
3734
use OCP\DB\QueryBuilder\IQueryBuilder;
3835
use OCP\Files\Notify\IChange;
3936
use OCP\Files\Notify\INotifyHandler;
4037
use OCP\Files\Notify\IRenameChange;
4138
use OCP\Files\Storage\INotifyStorage;
4239
use OCP\Files\Storage\IStorage;
43-
use OCP\Files\StorageNotAvailableException;
4440
use OCP\IDBConnection;
4541
use OCP\IUserManager;
4642
use Psr\Log\LoggerInterface;
@@ -49,14 +45,14 @@
4945
use Symfony\Component\Console\Input\InputOption;
5046
use Symfony\Component\Console\Output\OutputInterface;
5147

52-
class Notify extends Base {
48+
class Notify extends StorageAuthBase {
5349
public function __construct(
54-
private GlobalStoragesService $globalService,
5550
private IDBConnection $connection,
5651
private LoggerInterface $logger,
57-
private IUserManager $userManager
52+
GlobalStoragesService $globalService,
53+
IUserManager $userManager,
5854
) {
59-
parent::__construct();
55+
parent::__construct($globalService, $userManager);
6056
}
6157

6258
protected function configure(): void {
@@ -97,71 +93,12 @@ protected function configure(): void {
9793
parent::configure();
9894
}
9995

100-
private function getUserOption(InputInterface $input): ?string {
101-
if ($input->getOption('user')) {
102-
return (string)$input->getOption('user');
103-
}
104-
105-
return $_ENV['NOTIFY_USER'] ?? $_SERVER['NOTIFY_USER'] ?? null;
106-
}
107-
108-
private function getPasswordOption(InputInterface $input): ?string {
109-
if ($input->getOption('password')) {
110-
return (string)$input->getOption('password');
111-
}
112-
113-
return $_ENV['NOTIFY_PASSWORD'] ?? $_SERVER['NOTIFY_PASSWORD'] ?? null;
114-
}
115-
11696
protected function execute(InputInterface $input, OutputInterface $output): int {
117-
$mount = $this->globalService->getStorage($input->getArgument('mount_id'));
118-
if (is_null($mount)) {
119-
$output->writeln('<error>Mount not found</error>');
97+
[$mount, $storage] = $this->createStorage($input, $output);
98+
if ($storage === null) {
12099
return self::FAILURE;
121100
}
122-
$noAuth = false;
123-
124-
$userOption = $this->getUserOption($input);
125-
$passwordOption = $this->getPasswordOption($input);
126-
127-
// if only the user is provided, we get the user object to pass along to the auth backend
128-
// this allows using saved user credentials
129-
$user = ($userOption && !$passwordOption) ? $this->userManager->get($userOption) : null;
130-
131-
try {
132-
$authBackend = $mount->getAuthMechanism();
133-
$authBackend->manipulateStorageConfig($mount, $user);
134-
} catch (InsufficientDataForMeaningfulAnswerException $e) {
135-
$noAuth = true;
136-
} catch (StorageNotAvailableException $e) {
137-
$noAuth = true;
138-
}
139101

140-
if ($userOption) {
141-
$mount->setBackendOption('user', $userOption);
142-
}
143-
if ($passwordOption) {
144-
$mount->setBackendOption('password', $passwordOption);
145-
}
146-
147-
try {
148-
$backend = $mount->getBackend();
149-
$backend->manipulateStorageConfig($mount, $user);
150-
} catch (InsufficientDataForMeaningfulAnswerException $e) {
151-
$noAuth = true;
152-
} catch (StorageNotAvailableException $e) {
153-
$noAuth = true;
154-
}
155-
156-
try {
157-
$storage = $this->createStorage($mount);
158-
} catch (\Exception $e) {
159-
$output->writeln('<error>Error while trying to create storage</error>');
160-
if ($noAuth) {
161-
$output->writeln('<error>Login and/or password required</error>');
162-
}
163-
return self::FAILURE;
164-
}
165102
if (!$storage instanceof INotifyStorage) {
166103
$output->writeln('<error>Mount of type "' . $mount->getBackend()->getText() . '" does not support active update notifications</error>');
167104
return self::FAILURE;
@@ -189,11 +126,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int
189126
return self::SUCCESS;
190127
}
191128

192-
private function createStorage(StorageConfig $mount): IStorage {
193-
$class = $mount->getBackend()->getStorageClass();
194-
return new $class($mount->getBackendOptions());
195-
}
196-
197129
private function markParentAsOutdated($mountId, $path, OutputInterface $output, bool $dryRun): void {
198130
$parent = ltrim(dirname($path), '/');
199131
if ($parent === '.') {
+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* @copyright Copyright (c) 2021 Robin Appelman <robin@icewind.nl>
6+
*
7+
* @license GNU AGPL version 3 or any later version
8+
*
9+
* This program is free software: you can redistribute it and/or modify
10+
* it under the terms of the GNU Affero General Public License as
11+
* published by the Free Software Foundation, either version 3 of the
12+
* License, or (at your option) any later version.
13+
*
14+
* This program is distributed in the hope that it will be useful,
15+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
* GNU Affero General Public License for more details.
18+
*
19+
* You should have received a copy of the GNU Affero General Public License
20+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
21+
*
22+
*/
23+
24+
namespace OCA\Files_External\Command;
25+
26+
use OC\Files\Cache\Scanner;
27+
use OCA\Files_External\Service\GlobalStoragesService;
28+
use OCP\IUserManager;
29+
use Symfony\Component\Console\Helper\Table;
30+
use Symfony\Component\Console\Input\InputArgument;
31+
use Symfony\Component\Console\Input\InputInterface;
32+
use Symfony\Component\Console\Input\InputOption;
33+
use Symfony\Component\Console\Output\OutputInterface;
34+
35+
class Scan extends StorageAuthBase {
36+
protected float $execTime = 0;
37+
protected int $foldersCounter = 0;
38+
protected int $filesCounter = 0;
39+
40+
public function __construct(
41+
GlobalStoragesService $globalService,
42+
IUserManager $userManager
43+
) {
44+
parent::__construct($globalService, $userManager);
45+
}
46+
47+
protected function configure(): void {
48+
$this
49+
->setName('files_external:scan')
50+
->setDescription('Scan an external storage for changed files')
51+
->addArgument(
52+
'mount_id',
53+
InputArgument::REQUIRED,
54+
'the mount id of the mount to scan'
55+
)->addOption(
56+
'user',
57+
'u',
58+
InputOption::VALUE_REQUIRED,
59+
'The username for the remote mount (required only for some mount configuration that don\'t store credentials)'
60+
)->addOption(
61+
'password',
62+
'p',
63+
InputOption::VALUE_REQUIRED,
64+
'The password for the remote mount (required only for some mount configuration that don\'t store credentials)'
65+
)->addOption(
66+
'path',
67+
'',
68+
InputOption::VALUE_OPTIONAL,
69+
'The path in the storage to scan',
70+
''
71+
);
72+
parent::configure();
73+
}
74+
75+
protected function execute(InputInterface $input, OutputInterface $output): int {
76+
[, $storage] = $this->createStorage($input, $output);
77+
if ($storage === null) {
78+
return 1;
79+
}
80+
81+
$path = $input->getOption('path');
82+
83+
$this->execTime = -microtime(true);
84+
85+
/** @var Scanner $scanner */
86+
$scanner = $storage->getScanner();
87+
88+
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFile', function (string $path) use ($output) {
89+
$output->writeln("\tFile\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
90+
++$this->filesCounter;
91+
$this->abortIfInterrupted();
92+
});
93+
94+
$scanner->listen('\OC\Files\Cache\Scanner', 'scanFolder', function (string $path) use ($output) {
95+
$output->writeln("\tFolder\t<info>$path</info>", OutputInterface::VERBOSITY_VERBOSE);
96+
++$this->foldersCounter;
97+
$this->abortIfInterrupted();
98+
});
99+
100+
$scanner->scan($path);
101+
102+
$this->presentStats($output);
103+
104+
return 0;
105+
}
106+
107+
/**
108+
* @param OutputInterface $output
109+
*/
110+
protected function presentStats(OutputInterface $output): void {
111+
// Stop the timer
112+
$this->execTime += microtime(true);
113+
114+
$headers = [
115+
'Folders', 'Files', 'Elapsed time'
116+
];
117+
118+
$this->showSummary($headers, [], $output);
119+
}
120+
121+
/**
122+
* Shows a summary of operations
123+
*
124+
* @param string[] $headers
125+
* @param string[] $rows
126+
* @param OutputInterface $output
127+
*/
128+
protected function showSummary(array $headers, array $rows, OutputInterface $output): void {
129+
$niceDate = $this->formatExecTime();
130+
if (!$rows) {
131+
$rows = [
132+
$this->foldersCounter,
133+
$this->filesCounter,
134+
$niceDate,
135+
];
136+
}
137+
$table = new Table($output);
138+
$table
139+
->setHeaders($headers)
140+
->setRows([$rows]);
141+
$table->render();
142+
}
143+
144+
145+
/**
146+
* Formats microtime into a human readable format
147+
*
148+
* @return string
149+
*/
150+
protected function formatExecTime(): string {
151+
$secs = round($this->execTime);
152+
# convert seconds into HH:MM:SS form
153+
return sprintf('%02d:%02d:%02d', ($secs / 3600), ($secs / 60 % 60), $secs % 60);
154+
}
155+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* @copyright Copyright (c) 2021 Robin Appelman <robin@icewind.nl>
6+
*
7+
* @license GNU AGPL version 3 or any later version
8+
*
9+
* This program is free software: you can redistribute it and/or modify
10+
* it under the terms of the GNU Affero General Public License as
11+
* published by the Free Software Foundation, either version 3 of the
12+
* License, or (at your option) any later version.
13+
*
14+
* This program is distributed in the hope that it will be useful,
15+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
* GNU Affero General Public License for more details.
18+
*
19+
* You should have received a copy of the GNU Affero General Public License
20+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
21+
*
22+
*/
23+
24+
namespace OCA\Files_External\Command;
25+
26+
27+
use OC\Core\Command\Base;
28+
use OCA\Files_External\Lib\InsufficientDataForMeaningfulAnswerException;
29+
use OCA\Files_External\Lib\StorageConfig;
30+
use OCA\Files_External\NotFoundException;
31+
use OCA\Files_External\Service\GlobalStoragesService;
32+
use OCP\Files\Storage\IStorage;
33+
use OCP\Files\StorageNotAvailableException;
34+
use OCP\IUserManager;
35+
use Symfony\Component\Console\Input\InputInterface;
36+
use Symfony\Component\Console\Output\OutputInterface;
37+
38+
abstract class StorageAuthBase extends Base {
39+
public function __construct(
40+
protected GlobalStoragesService $globalService,
41+
protected IUserManager $userManager,
42+
) {
43+
parent::__construct();
44+
}
45+
46+
private function getUserOption(InputInterface $input): ?string {
47+
if ($input->getOption('user')) {
48+
return (string)$input->getOption('user');
49+
}
50+
51+
return $_ENV['NOTIFY_USER'] ?? $_SERVER['NOTIFY_USER'] ?? null;
52+
}
53+
54+
private function getPasswordOption(InputInterface $input): ?string {
55+
if ($input->getOption('password')) {
56+
return (string)$input->getOption('password');
57+
}
58+
59+
return $_ENV['NOTIFY_PASSWORD'] ?? $_SERVER['NOTIFY_PASSWORD'] ?? null;
60+
}
61+
62+
/**
63+
* @param InputInterface $input
64+
* @param OutputInterface $output
65+
* @return array
66+
* @psalm-return array{0: StorageConfig, 1: IStorage}|array{0: null, 1: null}
67+
*/
68+
protected function createStorage(InputInterface $input, OutputInterface $output): array {
69+
try {
70+
/** @var StorageConfig|null $mount */
71+
$mount = $this->globalService->getStorage($input->getArgument('mount_id'));
72+
} catch (NotFoundException $e) {
73+
$output->writeln('<error>Mount not found</error>');
74+
return [null, null];
75+
}
76+
if (is_null($mount)) {
77+
$output->writeln('<error>Mount not found</error>');
78+
return [null, null];
79+
}
80+
$noAuth = false;
81+
82+
$userOption = $this->getUserOption($input);
83+
$passwordOption = $this->getPasswordOption($input);
84+
85+
// if only the user is provided, we get the user object to pass along to the auth backend
86+
// this allows using saved user credentials
87+
$user = ($userOption && !$passwordOption) ? $this->userManager->get($userOption) : null;
88+
89+
try {
90+
$authBackend = $mount->getAuthMechanism();
91+
$authBackend->manipulateStorageConfig($mount, $user);
92+
} catch (InsufficientDataForMeaningfulAnswerException $e) {
93+
$noAuth = true;
94+
} catch (StorageNotAvailableException $e) {
95+
$noAuth = true;
96+
}
97+
98+
if ($userOption) {
99+
$mount->setBackendOption('user', $userOption);
100+
}
101+
if ($passwordOption) {
102+
$mount->setBackendOption('password', $passwordOption);
103+
}
104+
105+
try {
106+
$backend = $mount->getBackend();
107+
$backend->manipulateStorageConfig($mount, $user);
108+
} catch (InsufficientDataForMeaningfulAnswerException $e) {
109+
$noAuth = true;
110+
} catch (StorageNotAvailableException $e) {
111+
$noAuth = true;
112+
}
113+
114+
try {
115+
$class = $mount->getBackend()->getStorageClass();
116+
/** @var IStorage $storage */
117+
$storage = new $class($mount->getBackendOptions());
118+
if (!$storage->test()) {
119+
throw new \Exception();
120+
}
121+
return [$mount, $storage];
122+
} catch (\Exception $e) {
123+
$output->writeln('<error>Error while trying to create storage</error>');
124+
if ($noAuth) {
125+
$output->writeln('<error>Username and/or password required</error>');
126+
}
127+
return [null, null];
128+
}
129+
}
130+
}

0 commit comments

Comments
 (0)
Please sign in to comment.