Skip to content

Commit

Permalink
Merge #232 NC patch - multiple cron jobs, background job classes
Browse files Browse the repository at this point in the history
  • Loading branch information
@tsdicloud
tsdicloud committed Jun 20, 2023
2 parents c5d5739 + 8b3d037 commit 8d65c80
Show file tree
Hide file tree
Showing 5 changed files with 259 additions and 5 deletions.
221 changes: 221 additions & 0 deletions core/Command/Background/Worker.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,221 @@
<?php

declare(strict_types=1);
/**
* @copyright Copyright (c) 2021, Joas Schilling <coding@schilljs.com>
*
* @author Joas Schilling <coding@schilljs.com>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/

namespace OC\Core\Command\Background;

use OCP\BackgroundJob\IJob;
use OCP\BackgroundJob\IJobList;
use Psr\Log\LoggerInterface;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;

class Worker extends Command
{
protected IJobList $jobList;
protected LoggerInterface $logger;

public const DEFAULT_INTERVAL = 5;

public function __construct(
IJobList $jobList,
LoggerInterface $logger
) {
parent::__construct();
$this->jobList = $jobList;
$this->logger = $logger;
}

protected function configure(): void
{
$this
->setName('background-job:worker')
->setDescription('Run a background job worker')
->addArgument(
'job-class',
InputArgument::OPTIONAL,
'The class of the job in the database'
)
->addOption(
'once',
null,
InputOption::VALUE_NONE,
'Only execute the worker once (as a regular cron execution would do it)'
);
}

protected function execute(InputInterface $input, OutputInterface $output): int
{
$jobClass = $input->getArgument('job-class');

$executedJobs = [];

$ended = false;
pcntl_signal(SIGINT, function () use (&$ended, $output, &$executedJobs) {
$output->writeln('SIGINT');
if ($ended) {
foreach ($executedJobs as $id => $time) {
unset($executedJobs[$id]);
$job = $this->jobList->getById($id);
if ($job !== null) {
$this->jobList->unlockJob($job);
}
}
$output->writeln('<error>Killed');
exit(1);
}
$ended = true;
$output->writeln('<comment>Waiting for job to finish. Press Ctrl-C again to kill, but this may have unexpected side effects.</comment>');
});

while (true) {
if ($ended) {
break;
}
$count = 0;
$total = 0;
foreach ($this->jobList->countByClass() as $row) {
if ((int)$row['count'] === 1) {
$count++;
} else {
$output->writeln($row['class'] . " " . $row['count']);
}
$total += $row['count'];
}
$output->writeln("Other jobs " . $count);
$output->writeln("Total jobs " . $count);



foreach ($executedJobs as $id => $time) {
if ($time < time() - self::DEFAULT_INTERVAL) {
unset($executedJobs[$id]);
$job = $this->jobList->getById($id);
if ($job !== null) {
$this->jobList->unlockJob($job);
}
}
}

if ($input->getOption('once')) {
break;
}

$job = $this->jobList->getNext(false, $jobClass);
if (!$job) {
$output->writeln("Waiting for new jobs to be queued");
sleep(1);
continue;
}


if (isset($executedJobs[$job->getId()])) {
continue;
}

$output->writeln("- Running job " . get_class($job) . " " . $job->getId());

if ($output->isVerbose()) {
$this->printJobInfo($job->getId(), $job, $output);
}

$job->execute($this->jobList, \OC::$server->getLogger());

// clean up after unclean jobs
\OC_Util::tearDownFS();
\OC::$server->getTempManager()->clean();

$this->jobList->setLastJob($job);
$executedJobs[$job->getId()] = time();
unset($job);

if ($input->getOption('once')) {
break;
}
}

foreach ($executedJobs as $id => $time) {
unset($executedJobs[$id]);
$job = $this->jobList->getById($id);
if ($job !== null) {
$this->jobList->unlockJob($job);
}
}

return 0;
}

protected function printJobInfo(int $jobId, IJob $job, OutputInterface $output): void
{
$row = $this->jobList->getDetailsById($jobId);

$lastRun = new \DateTime();
$lastRun->setTimestamp((int) $row['last_run']);
$lastChecked = new \DateTime();
$lastChecked->setTimestamp((int) $row['last_checked']);
$reservedAt = new \DateTime();
$reservedAt->setTimestamp((int) $row['reserved_at']);

$output->writeln('Job class: ' . get_class($job));
$output->writeln('Arguments: ' . json_encode($job->getArgument()));

$isTimedJob = $job instanceof \OC\BackgroundJob\TimedJob || $job instanceof \OCP\BackgroundJob\TimedJob;
if ($isTimedJob) {
$output->writeln('Type: timed');
} elseif ($job instanceof \OC\BackgroundJob\QueuedJob || $job instanceof \OCP\BackgroundJob\QueuedJob) {
$output->writeln('Type: queued');
} else {
$output->writeln('Type: job');
}

$output->writeln('');
$output->writeln('Last checked: ' . $lastChecked->format(\DateTimeInterface::ATOM));
if ((int) $row['reserved_at'] === 0) {
$output->writeln('Reserved at: -');
} else {
$output->writeln('Reserved at: <comment>' . $reservedAt->format(\DateTimeInterface::ATOM) . '</comment>');
}
$output->writeln('Last executed: ' . $lastRun->format(\DateTimeInterface::ATOM));
$output->writeln('Last duration: ' . $row['execution_duration']);

if ($isTimedJob) {
$reflection = new \ReflectionClass($job);
$intervalProperty = $reflection->getProperty('interval');
$intervalProperty->setAccessible(true);
$interval = $intervalProperty->getValue($job);

$nextRun = new \DateTime();
$nextRun->setTimestamp($row['last_run'] + $interval);

if ($nextRun > new \DateTime()) {
$output->writeln('Next execution: <comment>' . $nextRun->format(\DateTimeInterface::ATOM) . '</comment>');
} else {
$output->writeln('Next execution: <info>' . $nextRun->format(\DateTimeInterface::ATOM) . '</info>');
}
}
}
}
2 changes: 2 additions & 0 deletions core/register_command.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@
$application->add(new OC\Core\Command\Background\Job(\OC::$server->getJobList(), \OC::$server->getLogger()));
$application->add(new OC\Core\Command\Background\ListCommand(\OC::$server->getJobList()));

$application->add(new OC\Core\Command\Background\Worker(\OC::$server->getJobList(), \OC::$server->get(LoggerInterface::class)));

$application->add(\OC::$server->query(\OC\Core\Command\Broadcast\Test::class));

$application->add(new OC\Core\Command\Config\App\DeleteConfig(\OC::$server->getConfig()));
Expand Down
3 changes: 2 additions & 1 deletion cron.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,8 @@
$endTime = time() + 14 * 60;

$executedJobs = [];
while ($job = $jobList->getNext($onlyTimeSensitive)) {
$jobClass = isset($argv[1]) ? $argv[1] : null;
while ($job = $jobList->getNext($onlyTimeSensitive, $jobClass)) {
if (isset($executedJobs[$job->getId()])) {
$jobList->unlockJob($job);
break;
Expand Down
28 changes: 25 additions & 3 deletions lib/private/BackgroundJob/JobList.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ public function getJobsIterator($job, ?int $limit, int $offset): iterable {
* Get the next job in the list
* @return ?IJob the next job to run. Beware that this object may be a singleton and may be modified by the next call to buildJob.
*/
public function getNext(bool $onlyTimeSensitive = false): ?IJob {
public function getNext(bool $onlyTimeSensitive = false, ?string $jobClass = null): ?IJob {
$query = $this->connection->getQueryBuilder();
$query->select('*')
->from('jobs')
Expand All @@ -218,6 +218,10 @@ public function getNext(bool $onlyTimeSensitive = false): ?IJob {
$query->andWhere($query->expr()->eq('time_sensitive', $query->createNamedParameter(IJob::TIME_SENSITIVE, IQueryBuilder::PARAM_INT)));
}

if ($jobClass) {
$query->andWhere($query->expr()->eq('class', $query->createNamedParameter($jobClass)));
}

$update = $this->connection->getQueryBuilder();
$update->update('jobs')
->set('reserved_at', $update->createNamedParameter($this->timeFactory->getTime()))
Expand All @@ -238,7 +242,7 @@ public function getNext(bool $onlyTimeSensitive = false): ?IJob {

if ($count === 0) {
// Background job already executed elsewhere, try again.
return $this->getNext($onlyTimeSensitive);
return $this->getNext($onlyTimeSensitive, $jobClass);
}
$job = $this->buildJob($row);

Expand All @@ -252,7 +256,7 @@ public function getNext(bool $onlyTimeSensitive = false): ?IJob {
$reset->executeStatement();

// Background job from disabled app, try again.
return $this->getNext($onlyTimeSensitive);
return $this->getNext($onlyTimeSensitive, $jobClass);
}

return $job;
Expand Down Expand Up @@ -374,6 +378,24 @@ public function setExecutionTime(IJob $job, $timeTaken): void {
$query->executeStatement();
}

public function countByClass(): array {
$query = $this->connection->getQueryBuilder();
$query->select('class')
->selectAlias($query->func()->count('id'), 'count')
->from('jobs')
->orderBy('count')
->groupBy('class');

$result = $query->executeQuery();

$jobs = [];
while ($row = $result->fetch()) {
$jobs[] = $row;
}

return $jobs;
}

/**
* Reset the $job so it executes on the next trigger
*
Expand Down
10 changes: 9 additions & 1 deletion lib/public/BackgroundJob/IJobList.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ public function getJobsIterator($job, ?int $limit, int $offset): iterable;
*
* @since 7.0.0 - In 24.0.0 parameter $onlyTimeSensitive got added
*/
public function getNext(bool $onlyTimeSensitive = false): ?IJob;
public function getNext(bool $onlyTimeSensitive = false, ?string $jobClass = null): ?IJob;

/**
* @since 7.0.0
Expand Down Expand Up @@ -146,6 +146,14 @@ public function setExecutionTime(IJob $job, int $timeTaken): void;
*/
public function resetBackgroundJob(IJob $job): void;

/**
* Count the number of job by class name
*
* @array{class-string<IJob>, int}
* @since 25.0.0
*/
public function countByClass(): array;

/**
* Checks whether a job of the passed class is reserved to run
*
Expand Down

0 comments on commit 8d65c80

Please sign in to comment.