Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(SpeechToText): Add SpeechToText OCP provider API #37674

Merged
merged 26 commits into from
Apr 19, 2023
Merged
Changes from 21 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
317521b
feat(SpeechToText): Add SpeechToText provider API
marcelklehr Apr 11, 2023
f2eb224
feat(SpeechToText): Add autoload changes
marcelklehr Apr 11, 2023
a9b5d1f
chore(coding style): Run cs:fix
marcelklehr Apr 11, 2023
1833d93
chore(coding style): Run cs:fix
marcelklehr Apr 11, 2023
ef7ce88
ISpeechToTextProvider#transcribeFile: Pass \OCP\Files\File instead of…
marcelklehr Apr 12, 2023
176f1af
ISpeechToTextManager: Take File as input, drop $context
marcelklehr Apr 13, 2023
a8d3fff
Split TranscriptionFinishedEvent into Successful and Failed events
marcelklehr Apr 13, 2023
865ebfa
Add missing strict_types + author + copyright
marcelklehr Apr 13, 2023
ad66180
Update lib/private/SpeechToText/SpeechToTextManager.php
marcelklehr Apr 13, 2023
71523b9
AbstractTranscriptionEvent: Add File param
marcelklehr Apr 17, 2023
ad1a0d8
Transcription*Event: Add doc block
marcelklehr Apr 17, 2023
3779cc3
SpeechToTextManager: Deduplicate transcription jobs
marcelklehr Apr 17, 2023
6e9f260
Update lib/private/SpeechToText/SpeechToTextManager.php
marcelklehr Apr 17, 2023
aac6a18
Update lib/public/SpeechToText/ISpeechToTextManager.php
marcelklehr Apr 17, 2023
b082657
Update lib/public/SpeechToText/ISpeechToTextManager.php
marcelklehr Apr 17, 2023
47cff5d
Update lib/public/SpeechToText/ISpeechToTextManager.php
marcelklehr Apr 17, 2023
3f57725
SpeechToTextManager#transcribeFile: Try next provider if one fails
marcelklehr Apr 17, 2023
bef4cf0
Fix missing import
nickvergessen Apr 18, 2023
9649f91
Update lib/private/SpeechToText/SpeechToTextManager.php
marcelklehr Apr 18, 2023
ce651e5
Run cs:fix
marcelklehr Apr 18, 2023
eb996cb
TranscriptionJob: Ensure filesystem is setup before trying to retriev…
marcelklehr Apr 18, 2023
a8b27c9
TranscriptionJob: Add owner argument to simplify filesystem setup
marcelklehr Apr 19, 2023
a2f5421
SpeechToTextManager#scheduleFileTranscription: Take context params an…
marcelklehr Apr 19, 2023
fbcd275
Context params: Make userId nullable
marcelklehr Apr 19, 2023
db9901a
Fix(docs): Fix parameter type in doc block
nickvergessen Apr 19, 2023
ab7b63d
fix(autoloader): Rebuild
nickvergessen Apr 19, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions lib/composer/composer/autoload_classmap.php
Original file line number Diff line number Diff line change
@@ -573,6 +573,9 @@
'OCP\\Share_Backend' => $baseDir . '/lib/public/Share_Backend.php',
'OCP\\Share_Backend_Collection' => $baseDir . '/lib/public/Share_Backend_Collection.php',
'OCP\\Share_Backend_File_Dependent' => $baseDir . '/lib/public/Share_Backend_File_Dependent.php',
'OCP\\SpeechToText\\Events\\TranscriptionFinishedEvent' => $baseDir . '/lib/public/SpeechToText/Events/TranscriptionFinishedEvent.php',
'OCP\\SpeechToText\\ISpeechToTextManager' => $baseDir . '/lib/public/SpeechToText/ISpeechToTextManager.php',
'OCP\\SpeechToText\\ISpeechToTextProvider' => $baseDir . '/lib/public/SpeechToText/ISpeechToTextProvider.php',
'OCP\\Support\\CrashReport\\ICollectBreadcrumbs' => $baseDir . '/lib/public/Support/CrashReport/ICollectBreadcrumbs.php',
'OCP\\Support\\CrashReport\\IMessageReporter' => $baseDir . '/lib/public/Support/CrashReport/IMessageReporter.php',
'OCP\\Support\\CrashReport\\IRegistry' => $baseDir . '/lib/public/Support/CrashReport/IRegistry.php',
@@ -1584,6 +1587,8 @@
'OC\\Share\\Constants' => $baseDir . '/lib/private/Share/Constants.php',
'OC\\Share\\Helper' => $baseDir . '/lib/private/Share/Helper.php',
'OC\\Share\\Share' => $baseDir . '/lib/private/Share/Share.php',
'OC\\SpeechToText\\SpeechToTextManager' => $baseDir . '/lib/private/SpeechToText/SpeechToTextManager.php',
'OC\\SpeechToText\\TranscriptionJob' => $baseDir . '/lib/private/SpeechToText/TranscriptionJob.php',
'OC\\StreamImage' => $baseDir . '/lib/private/StreamImage.php',
'OC\\Streamer' => $baseDir . '/lib/private/Streamer.php',
'OC\\SubAdmin' => $baseDir . '/lib/private/SubAdmin.php',
5 changes: 5 additions & 0 deletions lib/composer/composer/autoload_static.php
Original file line number Diff line number Diff line change
@@ -606,6 +606,9 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OCP\\Share_Backend' => __DIR__ . '/../../..' . '/lib/public/Share_Backend.php',
'OCP\\Share_Backend_Collection' => __DIR__ . '/../../..' . '/lib/public/Share_Backend_Collection.php',
'OCP\\Share_Backend_File_Dependent' => __DIR__ . '/../../..' . '/lib/public/Share_Backend_File_Dependent.php',
'OCP\\SpeechToText\\Events\\TranscriptionFinishedEvent' => __DIR__ . '/../../..' . '/lib/public/SpeechToText/Events/TranscriptionFinishedEvent.php',
'OCP\\SpeechToText\\ISpeechToTextManager' => __DIR__ . '/../../..' . '/lib/public/SpeechToText/ISpeechToTextManager.php',
'OCP\\SpeechToText\\ISpeechToTextProvider' => __DIR__ . '/../../..' . '/lib/public/SpeechToText/ISpeechToTextProvider.php',
'OCP\\Support\\CrashReport\\ICollectBreadcrumbs' => __DIR__ . '/../../..' . '/lib/public/Support/CrashReport/ICollectBreadcrumbs.php',
'OCP\\Support\\CrashReport\\IMessageReporter' => __DIR__ . '/../../..' . '/lib/public/Support/CrashReport/IMessageReporter.php',
'OCP\\Support\\CrashReport\\IRegistry' => __DIR__ . '/../../..' . '/lib/public/Support/CrashReport/IRegistry.php',
@@ -1617,6 +1620,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OC\\Share\\Constants' => __DIR__ . '/../../..' . '/lib/private/Share/Constants.php',
'OC\\Share\\Helper' => __DIR__ . '/../../..' . '/lib/private/Share/Helper.php',
'OC\\Share\\Share' => __DIR__ . '/../../..' . '/lib/private/Share/Share.php',
'OC\\SpeechToText\\SpeechToTextManager' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/SpeechToTextManager.php',
'OC\\SpeechToText\\TranscriptionJob' => __DIR__ . '/../../..' . '/lib/private/SpeechToText/TranscriptionJob.php',
'OC\\StreamImage' => __DIR__ . '/../../..' . '/lib/private/StreamImage.php',
'OC\\Streamer' => __DIR__ . '/../../..' . '/lib/private/Streamer.php',
'OC\\SubAdmin' => __DIR__ . '/../../..' . '/lib/private/SubAdmin.php',
22 changes: 22 additions & 0 deletions lib/private/AppFramework/Bootstrap/RegistrationContext.php
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@
use OCP\Calendar\Resource\IBackend as IResourceBackend;
use OCP\Calendar\Room\IBackend as IRoomBackend;
use OCP\Collaboration\Reference\IReferenceProvider;
use OCP\SpeechToText\ISpeechToTextProvider;
use OCP\Talk\ITalkBackend;
use OCP\Translation\ITranslationProvider;
use RuntimeException;
@@ -111,6 +112,9 @@ class RegistrationContext {
/** @var ServiceRegistration<IHandler>[] */
private $wellKnownHandlers = [];

/** @var ServiceRegistration<ISpeechToTextProvider>[] */
private $speechToTextProviders = [];

/** @var ServiceRegistration<ICustomTemplateProvider>[] */
private $templateProviders = [];

@@ -252,6 +256,13 @@ public function registerWellKnownHandler(string $class): void {
);
}

public function registerSpeechToTextProvider(string $providerClass): void {
$this->context->registerSpeechToTextProvider(
$this->appId,
$providerClass
);
}

public function registerTemplateProvider(string $providerClass): void {
$this->context->registerTemplateProvider(
$this->appId,
@@ -414,6 +425,10 @@ public function registerWellKnown(string $appId, string $class): void {
$this->wellKnownHandlers[] = new ServiceRegistration($appId, $class);
}

public function registerSpeechToTextProvider(string $appId, string $class): void {
$this->speechToTextProviders[] = new ServiceRegistration($appId, $class);
}

public function registerTemplateProvider(string $appId, string $class): void {
$this->templateProviders[] = new ServiceRegistration($appId, $class);
}
@@ -685,6 +700,13 @@ public function getWellKnownHandlers(): array {
return $this->wellKnownHandlers;
}

/**
* @return ServiceRegistration<ISpeechToTextProvider>[]
*/
public function getSpeechToTextProviders(): array {
return $this->speechToTextProviders;
}

/**
* @return ServiceRegistration<ICustomTemplateProvider>[]
*/
4 changes: 4 additions & 0 deletions lib/private/Server.php
Original file line number Diff line number Diff line change
@@ -148,6 +148,7 @@
use OC\Session\CryptoWrapper;
use OC\Share20\ProviderFactory;
use OC\Share20\ShareHelper;
use OC\SpeechToText\SpeechToTextManager;
use OC\SystemTag\ManagerFactory as SystemTagManagerFactory;
use OC\Tagging\TagMapper;
use OC\Talk\Broker;
@@ -246,6 +247,7 @@
use OCP\Security\ITrustedDomainHelper;
use OCP\Security\VerificationToken\IVerificationToken;
use OCP\Share\IShareHelper;
use OCP\SpeechToText\ISpeechToTextManager;
use OCP\SystemTag\ISystemTagManager;
use OCP\SystemTag\ISystemTagObjectMapper;
use OCP\Talk\IBroker;
@@ -1457,6 +1459,8 @@ public function __construct($webRoot, \OC\Config $config) {

$this->registerAlias(ITranslationManager::class, TranslationManager::class);

$this->registerAlias(ISpeechToTextManager::class, SpeechToTextManager::class);

$this->connectDispatcher();
}

119 changes: 119 additions & 0 deletions lib/private/SpeechToText/SpeechToTextManager.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
<?php

declare(strict_types=1);

/**
* @copyright Copyright (c) 2023 Julius Härtl <jus@bitgrid.net>
* @copyright Copyright (c) 2023 Marcel Klehr <mklehr@gmx.net>
*
* @author Julius Härtl <jus@bitgrid.net>
* @author Marcel Klehr <mklehr@gmx.net>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/


namespace OC\SpeechToText;

use InvalidArgumentException;
use OC\AppFramework\Bootstrap\Coordinator;
use OCP\BackgroundJob\IJobList;
use OCP\Files\File;
use OCP\Files\InvalidPathException;
use OCP\Files\NotFoundException;
use OCP\IServerContainer;
use OCP\PreConditionNotMetException;
use OCP\SpeechToText\ISpeechToTextManager;
use OCP\SpeechToText\ISpeechToTextProvider;
use Psr\Container\ContainerExceptionInterface;
use Psr\Container\NotFoundExceptionInterface;
use Psr\Log\LoggerInterface;
use RuntimeException;
use Throwable;

class SpeechToTextManager implements ISpeechToTextManager {
/** @var ?ISpeechToTextProvider[] */
private ?array $providers = null;

public function __construct(
private IServerContainer $serverContainer,
private Coordinator $coordinator,
private LoggerInterface $logger,
private IJobList $jobList,
) {
}

public function getProviders(): array {
$context = $this->coordinator->getRegistrationContext();
if ($context === null) {
return [];
}

if ($this->providers !== null) {
return $this->providers;
}

$this->providers = [];

foreach ($context->getSpeechToTextProviders() as $providerServiceRegistration) {
$class = $providerServiceRegistration->getService();
try {
$this->providers[$class] = $this->serverContainer->get($class);
} catch (NotFoundExceptionInterface|ContainerExceptionInterface|Throwable $e) {
$this->logger->error('Failed to load SpeechToText provider ' . $class, [
'exception' => $e,
]);
}
}

return $this->providers;
}

public function hasProviders(): bool {
$context = $this->coordinator->getRegistrationContext();
if ($context === null) {
return false;
}
return !empty($context->getSpeechToTextProviders());
}

public function scheduleFileTranscription(File $file): void {
marcelklehr marked this conversation as resolved.
Show resolved Hide resolved
if (!$this->hasProviders()) {
throw new PreConditionNotMetException('No SpeechToText providers have been registered');
}
try {
$this->jobList->add(TranscriptionJob::class, ['fileId' => $file->getId()]);
nickvergessen marked this conversation as resolved.
Show resolved Hide resolved
nickvergessen marked this conversation as resolved.
Show resolved Hide resolved
} catch (NotFoundException|InvalidPathException $e) {
throw new InvalidArgumentException('Invalid file provided for file transcription: ' . $e->getMessage());
}
}

public function transcribeFile(File $file): string {
if (!$this->hasProviders()) {
throw new PreConditionNotMetException('No SpeechToText providers have been registered');
}

foreach ($this->getProviders() as $provider) {
try {
return $provider->transcribeFile($file);
} catch (\Throwable $e) {
$this->logger->info('SpeechToText transcription using provider ' . $provider->getName() . ' failed', ['exception' => $e]);
}
}

throw new RuntimeException('Could not transcribe file');
Fixed Show fixed Hide fixed
nickvergessen marked this conversation as resolved.
Show resolved Hide resolved
}
}
142 changes: 142 additions & 0 deletions lib/private/SpeechToText/TranscriptionJob.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
<?php

declare(strict_types=1);

/**
* @copyright Copyright (c) 2023 Marcel Klehr <mklehr@gmx.net>
*
* @author Marcel Klehr <mklehr@gmx.net>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/


namespace OC\SpeechToText;
marcelklehr marked this conversation as resolved.
Show resolved Hide resolved

use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\QueuedJob;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\Config\ICachedMountFileInfo;
use OCP\Files\Config\IUserMountCache;
use OCP\Files\File;
use OCP\Files\IRootFolder;
use OCP\Files\Node;
use OCP\Files\NotFoundException;
use OCP\Files\NotPermittedException;
use OCP\PreConditionNotMetException;
use OCP\SpeechToText\Events\TranscriptionFailedEvent;
use OCP\SpeechToText\Events\TranscriptionSuccessfulEvent;
use OCP\SpeechToText\ISpeechToTextManager;
use Psr\Log\LoggerInterface;

class TranscriptionJob extends QueuedJob {
public function __construct(
ITimeFactory $timeFactory,
private ISpeechToTextManager $speechToTextManager,
private IEventDispatcher $eventDispatcher,
private IRootFolder $rootFolder,
private LoggerInterface $logger,
private IUserMountCache $userMountCache,
) {
parent::__construct($timeFactory);
}


/**
* @inheritDoc
*/
protected function run($argument) {
$fileId = $argument['fileId'];
nickvergessen marked this conversation as resolved.
Show resolved Hide resolved
$file = null;
try {
$file = $this->getFileFromId($fileId);
if (!($file instanceof File)) {
$this->logger->warning('Transcription of file ' . $fileId . ' failed. The file could not be found');
$this->eventDispatcher->dispatchTyped(
new TranscriptionFailedEvent(
$fileId,
null,
'File not found',
)
);
return;
}
$result = $this->speechToTextManager->transcribeFile($file);
$this->eventDispatcher->dispatchTyped(
new TranscriptionSuccessfulEvent(
$fileId,
$file,
$result,
)
);
} catch (PreConditionNotMetException|\RuntimeException|\InvalidArgumentException|NotFoundException $e) {
$this->logger->warning('Transcription of file ' . $fileId . ' failed', ['exception' => $e]);
$this->eventDispatcher->dispatchTyped(
new TranscriptionFailedEvent(
$fileId,
$file,
$e->getMessage(),
)
);
}
}

/**
* @throws NotFoundException
*/
private function getFileFromId(int $fileId): Node {
$mountPoints = $this->userMountCache->getMountsForFileId($fileId);
if (empty($mountPoints)) {
throw new NotFoundException("No mount points found for file $fileId");
}

foreach ($mountPoints as $mountPoint) {
try {
return $this->getCreatableNodeFromMountPoint($mountPoint, $fileId);
} catch (NotPermittedException $e) {
// Check the next mount point
$this->logger->debug('Mount point ' . ($mountPoint->getMountId() ?? 'null') . ' has no delete permissions for file ' . $fileId);
} catch (NotFoundException $e) {
// Already logged explicitly inside
}
}

throw new NotFoundException("No mount point with delete permissions found for file $fileId");
}

/**
* @throws NotFoundException
*/
protected function getCreatableNodeFromMountPoint(ICachedMountFileInfo $mountPoint, int $fileId): Node {
try {
$userId = $mountPoint->getUser()->getUID();
$userFolder = $this->rootFolder->getUserFolder($userId);
\OC_Util::setupFS($userId);
} catch (\Exception $e) {
$this->logger->debug($e->getMessage(), [
'exception' => $e,
]);
throw new NotFoundException('Could not get user', 0, $e);
}

$nodes = $userFolder->getById($fileId);
if (empty($nodes)) {
throw new NotFoundException('No node for file ' . $fileId . ' and user ' . $userId);
}

return array_shift($nodes);
}
}
Loading