feat(files_trashbin): Refactor expire background job to support parallel run

- Follow-up of #51600

The original PR introduced the possibility to continue an `ExpireTrash` job by saving the offset. This was to prevent having to start over the whole user list when the job crashed or was killed.

But on big instances, one process is not enough to go through all the users in a timely manner. Supporting parallel run allows covering more ground faster.

This PR introduced this possibility. We are now storing the offset right away to allow another parallel job to pick up the task at that point. We are arbitrarily cutting the user list in chunk of 10 to not drastically overflow the 30 minutes time limit.

Signed-off-by: Louis Chemineau <louis@chmn.me>

Author: artonge

apps/files_trashbin/lib/BackgroundJob/ExpireTrash.php

@@ -6,6 +6,7 @@
  */
 namespace OCA\Files_Trashbin\BackgroundJob;
 
+use OC\Files\SetupManager;
 use OC\Files\View;
 use OCA\Files_Trashbin\Expiration;
 use OCA\Files_Trashbin\Helper;
@@ -17,16 +18,18 @@
 use Psr\Log\LoggerInterface;
 
 class ExpireTrash extends TimedJob {
+	private const THIRTY_MINUTES = 30 * 60;
+
 	public function __construct(
 		private IAppConfig $appConfig,
 		private IUserManager $userManager,
 		private Expiration $expiration,
 		private LoggerInterface $logger,
+		private SetupManager $setupManager,
 		ITimeFactory $time,
 	) {
 		parent::__construct($time);
-		// Run once per 30 minutes
-		$this->setInterval(60 * 30);
+		$this->setInterval(self::THIRTY_MINUTES);
 	}
 
 	protected function run($argument) {
@@ -40,44 +43,47 @@ protected function run($argument) {
 			return;
 		}
 
-		$stopTime = time() + 60 * 30; // Stops after 30 minutes.
-		$offset = $this->appConfig->getValueInt('files_trashbin', 'background_job_expire_trash_offset', 0);
-		$users = $this->userManager->getSeenUsers($offset);
+		$stopTime = time() + self::THIRTY_MINUTES;
+
+		do {
+			$this->appConfig->clearCache();
+			$offset = $this->appConfig->getValueInt('files_trashbin', 'background_job_expire_trash_offset', 0);
+			$this->appConfig->setValueInt('files_trashbin', 'background_job_expire_trash_offset', $offset + 10);
+
+			$users = $this->userManager->getSeenUsers($offset, 10);
+			$count = 0;
+
+			foreach ($users as $uid) {
+				$uid = $uid->getUID();
+				$count++;
 
-		foreach ($users as $user) {
-			try {
-				$uid = $user->getUID();
-				if (!$this->setupFS($uid)) {
-					continue;
+				try {
+					if ($this->setupFS($uid)) {
+						$dirContent = Helper::getTrashFiles('/', $uid, 'mtime');
+						Trashbin::deleteExpiredFiles($dirContent, $uid);
+					}
+				} catch (\Throwable $e) {
+					$this->logger->error('Error while expiring trashbin for user ' . $uid, ['exception' => $e]);
 				}
-				$dirContent = Helper::getTrashFiles('/', $uid, 'mtime');
-				Trashbin::deleteExpiredFiles($dirContent, $uid);
-			} catch (\Throwable $e) {
-				$this->logger->error('Error while expiring trashbin for user ' . $user->getUID(), ['exception' => $e]);
+
+				$this->setupManager->tearDown();
 			}
 
-			$offset++;
+		} while (time() < $stopTime && $count === 10);
 
-			if ($stopTime < time()) {
-				$this->appConfig->setValueInt('files_trashbin', 'background_job_expire_trash_offset', $offset);
-				\OC_Util::tearDownFS();
-				return;
-			}
+		if ($count < 10) {
+			$this->appConfig->setValueInt('files_trashbin', 'background_job_expire_trash_offset', 0);
 		}
-
-		$this->appConfig->setValueInt('files_trashbin', 'background_job_expire_trash_offset', 0);
-		\OC_Util::tearDownFS();
 	}
 
 	/**
 	 * Act on behalf on trash item owner
 	 */
-	protected function setupFS(string $user): bool {
-		\OC_Util::tearDownFS();
-		\OC_Util::setupFS($user);
+	protected function setupFS(string $uid): bool {
+		$this->setupManager->setupForUser($this->userManager->get($uid));
 
 		// Check if this user has a trashbin directory
-		$view = new View('/' . $user);
+		$view = new View('/' . $uid);
 		if (!$view->is_dir('/files_trashbin/files')) {
 			return false;
 		}

apps/files_trashbin/tests/BackgroundJob/ExpireTrashTest.php

@@ -7,6 +7,7 @@
 
 namespace OCA\Files_Trashbin\Tests\BackgroundJob;
 
+use OC\Files\SetupManager;
 use OCA\Files_Trashbin\BackgroundJob\ExpireTrash;
 use OCA\Files_Trashbin\Expiration;
 use OCP\AppFramework\Utility\ITimeFactory;
@@ -18,23 +19,14 @@
 use Test\TestCase;
 
 class ExpireTrashTest extends TestCase {
-	/** @var IAppConfig&MockObject */
-	private $appConfig;
 
-	/** @var IUserManager&MockObject */
-	private $userManager;
-
-	/** @var Expiration&MockObject */
-	private $expiration;
-
-	/** @var IJobList&MockObject */
-	private $jobList;
-
-	/** @var LoggerInterface&MockObject */
-	private $logger;
-
-	/** @var ITimeFactory&MockObject */
-	private $time;
+	private IAppConfig&MockObject $appConfig;
+	private IUserManager&MockObject $userManager;
+	private Expiration&MockObject $expiration;
+	private IJobList&MockObject $jobList;
+	private LoggerInterface&MockObject $logger;
+	private ITimeFactory&MockObject $time;
+	private SetupManager&MockObject $setupManager;
 
 	protected function setUp(): void {
 		parent::setUp();
@@ -44,6 +36,7 @@ protected function setUp(): void {
 		$this->expiration = $this->createMock(Expiration::class);
 		$this->jobList = $this->createMock(IJobList::class);
 		$this->logger = $this->createMock(LoggerInterface::class);
+		$this->setupManager = $this->createMock(SetupManager::class);
 
 		$this->time = $this->createMock(ITimeFactory::class);
 		$this->time->method('getTime')
@@ -63,7 +56,7 @@ public function testConstructAndRun(): void {
 			->with('files_trashbin', 'background_job_expire_trash_offset', 0)
 			->willReturn(0);
 
-		$job = new ExpireTrash($this->appConfig, $this->userManager, $this->expiration, $this->logger, $this->time);
+		$job = new ExpireTrash($this->appConfig, $this->userManager, $this->expiration, $this->logger, $this->setupManager, $this->time);
 		$job->start($this->jobList);
 	}
 
@@ -74,7 +67,7 @@ public function testBackgroundJobDeactivated(): void {
 		$this->expiration->expects($this->never())
 			->method('getMaxAgeAsTimestamp');
 
-		$job = new ExpireTrash($this->appConfig, $this->userManager, $this->expiration, $this->logger, $this->time);
+		$job = new ExpireTrash($this->appConfig, $this->userManager, $this->expiration, $this->logger, $this->setupManager, $this->time);
 		$job->start($this->jobList);
 	}
 }

build/psalm-baseline.xml

@@ -1792,19 +1792,6 @@
       <code><![CDATA[isset($_['hideFileList']) && $_['hideFileList'] !== true]]></code>
     </RedundantCondition>
   </file>
-  <file src="apps/files_trashbin/lib/BackgroundJob/ExpireTrash.php">
-    <DeprecatedClass>
-      <code><![CDATA[\OC_Util::setupFS($user)]]></code>
-      <code><![CDATA[\OC_Util::tearDownFS()]]></code>
-      <code><![CDATA[\OC_Util::tearDownFS()]]></code>
-      <code><![CDATA[\OC_Util::tearDownFS()]]></code>
-    </DeprecatedClass>
-    <DeprecatedMethod>
-      <code><![CDATA[\OC_Util::tearDownFS()]]></code>
-      <code><![CDATA[\OC_Util::tearDownFS()]]></code>
-      <code><![CDATA[\OC_Util::tearDownFS()]]></code>
-    </DeprecatedMethod>
-  </file>
   <file src="apps/files_trashbin/lib/Command/CleanUp.php">
     <DeprecatedClass>
       <code><![CDATA[\OC_Util::setupFS($uid)]]></code>

lib/private/User/Manager.php

@@ -634,7 +634,7 @@ public function callForSeenUsers(\Closure $callback) {
 	}
 
 	/**
-	 * Getting all userIds that have a listLogin value requires checking the
+	 * Getting all userIds that have a lastLogin value requires checking the
 	 * value in php because on oracle you cannot use a clob in a where clause,
 	 * preventing us from doing a not null or length(value) > 0 check.
 	 *
@@ -812,19 +812,17 @@ public function getDisplayNameCache(): DisplayNameCache {
 		return $this->displayNameCache;
 	}
 
-	/**
-	 * Gets the list of users sorted by lastLogin, from most recent to least recent
-	 *
-	 * @param int $offset from which offset to fetch
-	 * @return \Iterator<IUser> list of user IDs
-	 * @since 30.0.0
-	 */
-	public function getSeenUsers(int $offset = 0): \Iterator {
-		$limit = 1000;
-
+	public function getSeenUsers(int $offset = 0, ?int $limit = null): \Iterator {
 		do {
-			$userIds = $this->getSeenUserIds($limit, $offset);
-			$offset += $limit;
+			if ($limit !== null) {
+				$batchSize = min($limit, 1000);
+				$limit -= $batchSize;
+			} else {
+				$batchSize = 1000;
+			}
+
+			$userIds = $this->getSeenUserIds($batchSize, $offset);
+			$offset += $batchSize;
 
 			foreach ($userIds as $userId) {
 				foreach ($this->backends as $backend) {
@@ -835,6 +833,6 @@ public function getSeenUsers(int $offset = 0): \Iterator {
 					}
 				}
 			}
-		} while (count($userIds) === $limit);
+		} while (count($userIds) === $batchSize && $limit !== 0);
 	}
 }

lib/public/IUserManager.php

@@ -239,8 +239,9 @@ public function getLastLoggedInUsers(?int $limit = null, int $offset = 0, string
 	 * The offset argument allows the caller to continue the iteration at a specific offset.
 	 *
 	 * @param int $offset from which offset to fetch
+	 * @param int|null $limit maximum number of records to fetch
 	 * @return \Iterator<IUser> list of IUser object
 	 * @since 32.0.0
 	 */
-	public function getSeenUsers(int $offset = 0): \Iterator;
+	public function getSeenUsers(int $offset = 0, ?int $limit = null): \Iterator;
 }