Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Truncate tables and rename documents folder on reset #5918

Merged
merged 3 commits into from
Jun 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions lib/Command/ResetDocument.php
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,17 @@ protected function execute(InputInterface $input, OutputInterface $output): int
return 1;
}

if ($all && $fullReset) {
// Truncate tables and clear document directory
$this->documentService->clearAll();
return 0;
}

if ($all) {
$fileIds = array_map(static function (Document $document) {
return $document->getId();
}, $this->documentService->getAll());
$fileIds = [];
foreach ($this->documentService->getAll() as $document) {
$fileIds[] = $document->getId();
}
} else {
$fileIds = [$fileId];
}
Expand Down
9 changes: 5 additions & 4 deletions lib/Cron/Cleanup.php
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,8 @@ public function __construct(ITimeFactory $time,
*/
protected function run($argument): void {
$this->logger->debug('Run cleanup job for text documents');
$documents = $this->documentService->getAll();
foreach ($documents as $document) {
$allSessions = $this->sessionService->getAllSessions($document->getId());
if (count($allSessions) > 0) {
foreach ($this->documentService->getAll() as $document) {
if ($this->sessionService->countAllSessions($document->getId()) > 0) {
// Do not reset if there are any sessions left
// Inactive sessions will get removed further down and will trigger a reset next time
continue;
Expand All @@ -57,5 +55,8 @@ protected function run($argument): void {
$this->logger->debug('Run cleanup job for text sessions');
$removedSessions = $this->sessionService->removeInactiveSessionsWithoutSteps(null);
$this->logger->debug('Removed ' . $removedSessions . ' inactive sessions');

$this->logger->debug('Run cleanup job for obsolete documents folders');
$this->documentService->cleanupOldDocumentsFolders();
}
}
26 changes: 24 additions & 2 deletions lib/Db/DocumentMapper.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

namespace OCA\Text\Db;

use Generator;
use OCP\AppFramework\Db\DoesNotExistException;
use OCP\AppFramework\Db\QBMapper;
use OCP\DB\QueryBuilder\IQueryBuilder;
Expand Down Expand Up @@ -38,12 +39,33 @@ public function find(int $documentId): Document {
return Document::fromRow($data);
}

public function findAll(): array {
public function findAll(): Generator {
$qb = $this->db->getQueryBuilder();
$result = $qb->select('*')
->from($this->getTableName())
->executeQuery();
try {
while ($row = $result->fetch()) {
yield $this->mapRowToEntity($row);
}
} finally {
$result->closeCursor();
}
}

return $this->findEntities($qb);
public function countAll(): int {
$qb = $this->db->getQueryBuilder();
$qb->select($qb->func()->count('id'))
->from($this->getTableName());
$result = $qb->executeQuery();
$count = (int)$result->fetchOne();
$result->closeCursor();
return $count;
}

public function clearAll(): void {
$qb = $this->db->getQueryBuilder();
$qb->delete($this->getTableName())
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

truncate and delete are different things. Truncate might be faster than delete, but not all DB engines support it afaik. Not sure what is the current state in doctrine. OK for now.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tested with 120.000 entries and it was super fast. So I guess it's ok for now 😬

Copy link
Member

@blizzz blizzz Jun 17, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

->executeStatement();
}
}
17 changes: 17 additions & 0 deletions lib/Db/SessionMapper.php
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,17 @@ public function findAll(int $documentId): array {
return $this->findEntities($qb);
}

public function countAll(int $documentId): int {
$qb = $this->db->getQueryBuilder();
$qb->select('id', 'color', 'document_id', 'last_awareness_message', 'last_contact', 'user_id', 'guest_name')
->from($this->getTableName())
->where($qb->expr()->eq('document_id', $qb->createNamedParameter($documentId)));
$result = $qb->executeQuery();
$count = (int)$result->fetchOne();
$result->closeCursor();
return $count;
}

/**
* @return Session[]
*
Expand Down Expand Up @@ -137,6 +148,12 @@ public function deleteByDocumentId(int $documentId): int {
return $qb->executeStatement();
}

public function clearAll(): void {
$qb = $this->db->getQueryBuilder();
$qb->delete($this->getTableName())
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

as above

->executeStatement();
}

public function isUserInDocument(int $documentId, string $userId): bool {
$qb = $this->db->getQueryBuilder();
$result = $qb->select('*')
Expand Down
6 changes: 6 additions & 0 deletions lib/Db/StepMapper.php
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,12 @@ public function deleteAll(int $documentId): void {
->executeStatement();
}

public function clearAll(): void {
$qb = $this->db->getQueryBuilder();
$qb->delete($this->getTableName())
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

as above

->executeStatement();
}

// not in use right now
public function deleteBeforeVersion(int $documentId, int $version): int {
$qb = $this->db->getQueryBuilder();
Expand Down
17 changes: 1 addition & 16 deletions lib/Migration/ResetSessionsBeforeYjs.php
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@

namespace OCA\Text\Migration;

use OCA\Text\Db\Document;
use OCA\Text\Service\DocumentService;
use OCP\IAppConfig;
use OCP\Migration\IOutput;
Expand All @@ -32,20 +31,6 @@ public function run(IOutput $output): void {
return;
}

$fileIds = array_map(static function (Document $document) {
return $document->getId();
}, $this->documentService->getAll());

if (!$fileIds) {
return;
}

$output->startProgress(count($fileIds));
foreach ($fileIds as $fileId) {
$this->documentService->unlock($fileId);
$this->documentService->resetDocument($fileId, true);
$output->advance();
}
$output->finishProgress();
$this->documentService->clearAll();
}
}
43 changes: 41 additions & 2 deletions lib/Service/DocumentService.php
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
use OCP\Files\SimpleFS\ISimpleFile;
use OCP\ICache;
use OCP\ICacheFactory;
use OCP\IConfig;
use OCP\IRequest;
use OCP\Lock\LockedException;
use OCP\PreConditionNotMetException;
Expand Down Expand Up @@ -70,8 +71,9 @@ class DocumentService {
private IAppData $appData;
private ILockManager $lockManager;
private IUserMountCache $userMountCache;
private IConfig $config;

public function __construct(DocumentMapper $documentMapper, StepMapper $stepMapper, SessionMapper $sessionMapper, IAppData $appData, ?string $userId, IRootFolder $rootFolder, ICacheFactory $cacheFactory, LoggerInterface $logger, ShareManager $shareManager, IRequest $request, IManager $directManager, ILockManager $lockManager, IUserMountCache $userMountCache) {
public function __construct(DocumentMapper $documentMapper, StepMapper $stepMapper, SessionMapper $sessionMapper, IAppData $appData, ?string $userId, IRootFolder $rootFolder, ICacheFactory $cacheFactory, LoggerInterface $logger, ShareManager $shareManager, IRequest $request, IManager $directManager, ILockManager $lockManager, IUserMountCache $userMountCache, IConfig $config) {
$this->documentMapper = $documentMapper;
$this->stepMapper = $stepMapper;
$this->sessionMapper = $sessionMapper;
Expand All @@ -83,6 +85,7 @@ public function __construct(DocumentMapper $documentMapper, StepMapper $stepMapp
$this->shareManager = $shareManager;
$this->lockManager = $lockManager;
$this->userMountCache = $userMountCache;
$this->config = $config;
$token = $request->getParam('token');
if ($this->userId === null && $token !== null) {
try {
Expand Down Expand Up @@ -428,7 +431,7 @@ public function resetDocument(int $documentId, bool $force = false): void {
}
}

public function getAll(): array {
public function getAll(): \Generator {
return $this->documentMapper->findAll();
}

Expand Down Expand Up @@ -642,4 +645,40 @@ public function unlock(int $fileId): void {
} catch (NoLockProviderException | PreConditionNotMetException | NotFoundException $e) {
}
}

public function countAll(): int {
return $this->documentMapper->countAll();
}

private function getFullAppFolder(): Folder {
$appFolder = $this->rootFolder->get('appdata_' . $this->config->getSystemValueString('instanceid', '') . '/text');
if (!$appFolder instanceof Folder) {
throw new NotFoundException('Folder not found');
}
return $appFolder;
}

public function clearAll(): void {
$this->stepMapper->clearAll();
$this->sessionMapper->clearAll();
$this->documentMapper->clearAll();
try {
$appFolder = $this->getFullAppFolder();
$appFolder->get('documents')->move($appFolder->getPath() . '/documents_old_' . time());
} catch (NotFoundException) {
}
$this->ensureDocumentsFolder();
}

public function cleanupOldDocumentsFolders(): void {
try {
$appFolder = $this->getFullAppFolder();
foreach ($appFolder->getDirectoryListing() as $node) {
if (str_starts_with($node->getName(), 'documents_old_')) {
$node->delete();
}
}
} catch (NotFoundException) {
}
}
}
4 changes: 4 additions & 0 deletions lib/Service/SessionService.php
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,10 @@ public function getAllSessions(int $documentId): array {
}, $sessions);
}

public function countAllSessions(int $documentId): int {
return $this->sessionMapper->countAll($documentId);
}

public function getActiveSessions(int $documentId): array {
$sessions = $this->sessionMapper->findAllActive($documentId);
return array_map(function (Session $session) {
Expand Down
3 changes: 3 additions & 0 deletions tests/unit/Service/DocumentServiceTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
use OCP\Files\Lock\ILockManager;
use OCP\Files\NotPermittedException;
use OCP\ICacheFactory;
use OCP\IConfig;
use OCP\IRequest;
use OCP\Share\IManager;
use Psr\Log\LoggerInterface;
Expand Down Expand Up @@ -49,6 +50,7 @@ public function setUp(): void {
$this->directManager = $this->createMock(\OCP\DirectEditing\IManager::class);
$this->lockManager = $this->createMock(ILockManager::class);
$this->userMountCache = $this->createMock(IUserMountCache::class);
$config = $this->createMock(IConfig::class);

$this->documentService = new DocumentService(
$this->documentMapper,
Expand All @@ -64,6 +66,7 @@ public function setUp(): void {
$this->directManager,
$this->lockManager,
$this->userMountCache,
$config,
);
}

Expand Down
Loading