From 3929f66dedd76743eb33ca0353f5471c4390caca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Fri, 29 Apr 2016 17:36:28 +0200 Subject: [PATCH] Adding background:queue commands: worker, status and delete --- core/Command/Background/Queue/Delete.php | 45 ++++ core/Command/Background/Queue/Status.php | 41 ++++ core/Command/Background/Queue/Worker.php | 65 ++++++ core/register_command.php | 3 + db_structure.xml | 1 - lib/private/BackgroundJob/JobList.php | 26 ++- lib/private/Console/CommandLogger.php | 204 ++++++++++++++++++ lib/public/BackgroundJob/IJobList.php | 14 ++ .../Command/Background/Queue/DeleteTest.php | 72 +++++++ .../Command/Background/Queue/StatusTest.php | 72 +++++++ 10 files changed, 538 insertions(+), 5 deletions(-) create mode 100644 core/Command/Background/Queue/Delete.php create mode 100644 core/Command/Background/Queue/Status.php create mode 100644 core/Command/Background/Queue/Worker.php create mode 100644 lib/private/Console/CommandLogger.php create mode 100644 tests/Core/Command/Background/Queue/DeleteTest.php create mode 100644 tests/Core/Command/Background/Queue/StatusTest.php diff --git a/core/Command/Background/Queue/Delete.php b/core/Command/Background/Queue/Delete.php new file mode 100644 index 000000000000..e854092e6b9e --- /dev/null +++ b/core/Command/Background/Queue/Delete.php @@ -0,0 +1,45 @@ +jobList = $jobList; + parent::__construct(); + } + + protected function configure() { + $this + ->setName('background:queue:delete') + ->setDescription('Delete a job from the queue') + ->addArgument('id', InputArgument::REQUIRED, 'id of the job to be deleted'); + } + + /** + * @param InputInterface $input + * @param OutputInterface $output + * @return void + */ + protected function execute(InputInterface $input, OutputInterface $output) { + $id = $input->getArgument('id'); + + $job = $this->jobList->getById($id); + if ($job === null) { + $output->writeln("Job with id <$id> is not known."); + return; + } + + $this->jobList->removeById($id); + $output->writeln('Job has been deleted.'); + } +} diff --git a/core/Command/Background/Queue/Status.php b/core/Command/Background/Queue/Status.php new file mode 100644 index 000000000000..ec07c3b283db --- /dev/null +++ b/core/Command/Background/Queue/Status.php @@ -0,0 +1,41 @@ +jobList = $jobList; + parent::__construct(); + } + + protected function configure() { + $this + ->setName('background:queue:status') + ->setDescription('List queue status'); + } + + /** + * @param InputInterface $input + * @param OutputInterface $output + * @return void + */ + protected function execute(InputInterface $input, OutputInterface $output) { + $t = new Table($output); + $t->setHeaders(['Id', 'Job', 'Last run', 'Arguments']); + $this->jobList->listJobs(function (IJob $job) use ($t) { + $t->addRow([$job->getId(), \get_class($job), \date('c', $job->getLastRun()), $job->getArgument()]); + }); + $t->render(); + } +} diff --git a/core/Command/Background/Queue/Worker.php b/core/Command/Background/Queue/Worker.php new file mode 100644 index 000000000000..fae09f26963c --- /dev/null +++ b/core/Command/Background/Queue/Worker.php @@ -0,0 +1,65 @@ +jobList = \OC::$server->getJobList(); + parent::__construct(); + } + + protected function configure() { + $this + ->setName("background:queue:worker") + ->setDescription("Listen to the background job queue and execute the jobs") + ->addOption('sleep', null, InputOption::VALUE_OPTIONAL, 'Number of seconds to sleep when no job is available', 3); + } + + /** + * @param InputInterface $input + * @param OutputInterface $output + */ + protected function execute(InputInterface $input, OutputInterface $output) { + $this->logger = new CommandLogger($output); + + $waitTime = \max(1, $input->getOption('sleep')); + while (true) { + if ($this->hasBeenInterrupted()) { + break; + } + if ($this->executeNext() === null) { + \sleep($waitTime); + } + } + } + + private function executeNext() { + $job = $this->jobList->getNext(); + if ($job === null) { + return null; + } + $jobId = $job->getId(); + $this->logger->debug('Run job with ID ' . $job->getId(), ['app' => 'cron']); + $job->execute($this->jobList, $this->logger); + $this->logger->debug('Finished job with ID ' . $job->getId(), ['app' => 'cron']); + + $this->jobList->setLastJob($job); + unset($job); + + return $jobId; + } +} diff --git a/core/register_command.php b/core/register_command.php index 4f6aa9c13272..a5b6d22160a8 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -71,6 +71,9 @@ $application->add(new OC\Core\Command\Background\Cron(\OC::$server->getConfig())); $application->add(new OC\Core\Command\Background\WebCron(\OC::$server->getConfig())); $application->add(new OC\Core\Command\Background\Ajax(\OC::$server->getConfig())); + $application->add(new OC\Core\Command\Background\Queue\Worker()); + $application->add(new OC\Core\Command\Background\Queue\Status(\OC::$server->getJobList())); + $application->add(new OC\Core\Command\Background\Queue\Delete(\OC::$server->getJobList())); $application->add(new OC\Core\Command\Config\App\DeleteConfig(\OC::$server->getConfig())); $application->add(new OC\Core\Command\Config\App\GetConfig(\OC::$server->getConfig())); diff --git a/db_structure.xml b/db_structure.xml index 3043561f040b..d2666aee53c4 100644 --- a/db_structure.xml +++ b/db_structure.xml @@ -724,7 +724,6 @@ false - last_checked diff --git a/lib/private/BackgroundJob/JobList.php b/lib/private/BackgroundJob/JobList.php index 23f7bbe14e85..1a57d4b07282 100644 --- a/lib/private/BackgroundJob/JobList.php +++ b/lib/private/BackgroundJob/JobList.php @@ -108,7 +108,7 @@ public function remove($job, $argument = null) { /** * @param int $id */ - protected function removeById($id) { + public function removeById($id) { $query = $this->connection->getQueryBuilder(); $query->delete('jobs') ->where($query->expr()->eq('id', $query->createNamedParameter($id, IQueryBuilder::PARAM_INT))); @@ -305,9 +305,7 @@ public function getLastJob() { } /** - * set the lastRun of $job to now - * - * @param IJob $job + * @inheritdoc */ public function setLastRun($job) { $query = $this->connection->getQueryBuilder(); @@ -324,4 +322,24 @@ public function setExecutionTime($job, $timeTaken) { ->where($query->expr()->eq('id', $query->createNamedParameter($job->getId(), IQueryBuilder::PARAM_INT))); $query->execute(); } + + /** + * @inheritdoc + */ + public function listJobs(\Closure $callback) { + $query = $this->connection->getQueryBuilder(); + $query->select('*') + ->from('jobs'); + $result = $query->execute(); + + while ($row = $result->fetch()) { + $job = $this->buildJob($row); + if ($job) { + if ($callback($job) === false) { + break; + } + } + } + $result->closeCursor(); + } } diff --git a/lib/private/Console/CommandLogger.php b/lib/private/Console/CommandLogger.php new file mode 100644 index 000000000000..065ca1746f74 --- /dev/null +++ b/lib/private/Console/CommandLogger.php @@ -0,0 +1,204 @@ + + * + * @copyright Copyright (c) 2016, ownCloud, Inc. + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC\Console; + +use OCP\ILogger; +use OCP\Util; +use Symfony\Component\Console\Output\OutputInterface; + +class CommandLogger implements ILogger { + + /** @var OutputInterface */ + private $output; + + /** + * CommandLogger constructor. + * + * @param OutputInterface $output + */ + public function __construct(OutputInterface $output) { + $this->output = $output; + } + + /** + * System is unusable. + * + * @param string $message + * @param array $context + * @return null + * @since 7.0.0 + */ + public function emergency($message, array $context = []) { + $this->log(Util::FATAL, $message, $context); + } + + /** + * Action must be taken immediately. + * + * @param string $message + * @param array $context + * @return null + * @since 7.0.0 + */ + public function alert($message, array $context = []) { + $this->log(Util::ERROR, $message, $context); + } + + /** + * Critical conditions. + * + * @param string $message + * @param array $context + * @return null + * @since 7.0.0 + */ + public function critical($message, array $context = []) { + $this->log(Util::ERROR, $message, $context); + } + + /** + * Runtime errors that do not require immediate action but should typically + * be logged and monitored. + * + * @param string $message + * @param array $context + * @return null + * @since 7.0.0 + */ + public function error($message, array $context = []) { + $this->log(Util::ERROR, $message, $context); + } + + /** + * Exceptional occurrences that are not errors. + * + * @param string $message + * @param array $context + * @return null + * @since 7.0.0 + */ + public function warning($message, array $context = []) { + $this->log(Util::WARN, $message, $context); + } + + /** + * Normal but significant events. + * + * @param string $message + * @param array $context + * @return null + * @since 7.0.0 + */ + public function notice($message, array $context = []) { + $this->log(Util::INFO, $message, $context); + } + + /** + * Interesting events. + * + * @param string $message + * @param array $context + * @return null + * @since 7.0.0 + */ + public function info($message, array $context = []) { + $this->log(Util::ERROR, $message, $context); + } + + /** + * Detailed debug information. + * + * @param string $message + * @param array $context + * @return null + * @since 7.0.0 + */ + public function debug($message, array $context = []) { + $this->log(Util::DEBUG, $message, $context); + } + + /** + * Logs with an arbitrary level. + * + * @param mixed $level + * @param string $message + * @param array $context + * @return mixed + * @since 7.0.0 + */ + public function log($level, $message, array $context = []) { + $minLevel = Util::INFO; + $verbosity = $this->output->getVerbosity(); + if ($verbosity === OutputInterface::VERBOSITY_DEBUG) { + $minLevel = Util::DEBUG; + } + if ($verbosity === OutputInterface::VERBOSITY_QUIET) { + $minLevel = Util::ERROR; + } + if ($level < $minLevel) { + return; + } + + // interpolate $message as defined in PSR-3 + $replace = []; + foreach ($context as $key => $val) { + $replace['{' . $key . '}'] = $val; + } + + // interpolate replacement values into the message and return + $message = \strtr($message, $replace); + $style = ($level > 2) ?'error' : 'info'; + + $this->output->writeln("<$style>$message"); + } + + /** + * Logs an exception very detailed + * An additional message can we written to the log by adding it to the + * context. + * + * + * $logger->logException($ex, [ + * 'message' => 'Exception during cron job execution' + * ]); + * + * + * @param \Exception | \Throwable $exception + * @param array $context + * @return void + * @since 8.2.0 + */ + public function logException($exception, array $context = []) { + $exception = [ + 'Exception' => \get_class($exception), + 'Message' => $exception->getMessage(), + 'Code' => $exception->getCode(), + 'Trace' => $exception->getTraceAsString(), + 'File' => $exception->getFile(), + 'Line' => $exception->getLine(), + ]; + $exception['Trace'] = \preg_replace('!(login|checkPassword|updatePrivateKeyPassword)\(.*\)!', '$1(*** username and password replaced ***)', $exception['Trace']); + $msg = isset($context['message']) ? $context['message'] : 'Exception'; + $msg .= ': ' . \json_encode($exception); + $this->error($msg, $context); + } +} diff --git a/lib/public/BackgroundJob/IJobList.php b/lib/public/BackgroundJob/IJobList.php index 0120fb0ecbc7..e86b664d9697 100644 --- a/lib/public/BackgroundJob/IJobList.php +++ b/lib/public/BackgroundJob/IJobList.php @@ -127,4 +127,18 @@ public function setLastRun($job); * @since 10.0.0 */ public function setExecutionTime($job, $timeTaken); + + /** + * iterate over all jobs in the queue + * + * @return void + * @since 10.0.3 + */ + public function listJobs(\Closure $callback); + + /** + * remove a specific job by id + * @return void + * @since 10.0.9 */ + public function removeById($id); } diff --git a/tests/Core/Command/Background/Queue/DeleteTest.php b/tests/Core/Command/Background/Queue/DeleteTest.php new file mode 100644 index 000000000000..40c18ccdd6b4 --- /dev/null +++ b/tests/Core/Command/Background/Queue/DeleteTest.php @@ -0,0 +1,72 @@ + + * + * @copyright Copyright (c) 2017, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace Tests\Core\Command\Background\Queue; + +use OC\Core\Command\Background\Queue\Delete; +use OCP\BackgroundJob\IJobList; +use Symfony\Component\Console\Tester\CommandTester; +use Test\TestCase; + +/** + * Class DeleteTest + * + * @group DB + */ +class DeleteTest extends TestCase { + + /** @var CommandTester */ + private $commandTester; + /** @var IJobList */ + private $jobList; + + public function setUp() { + parent::setUp(); + + $this->jobList = $this->createMock(IJobList::class); + $this->jobList->expects($this->any())->method('getById') + ->willReturnCallback(function ($id) { + return ($id !== '666') ? true : null; + }); + + $command = new Delete($this->jobList); + $this->commandTester = new CommandTester($command); + } + + /** + * @dataProvider providesJobIds + * @param $jobId + * @param $expectedOutput + */ + public function testCommandInput($jobId, $expectedOutput) { + $input = ['id' => $jobId]; + $this->commandTester->execute($input); + $output = $this->commandTester->getDisplay(); + $this->assertContains($expectedOutput, $output); + } + + public function providesJobIds() { + return [ + ['666', 'Job with id <666> is not known.'], + ['1', 'Job has been deleted.'], + ]; + } +} diff --git a/tests/Core/Command/Background/Queue/StatusTest.php b/tests/Core/Command/Background/Queue/StatusTest.php new file mode 100644 index 000000000000..2fcf3f7a7134 --- /dev/null +++ b/tests/Core/Command/Background/Queue/StatusTest.php @@ -0,0 +1,72 @@ + + * + * @copyright Copyright (c) 2017, ownCloud GmbH + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace Tests\Core\Command\Background\Queue; + +use OC\BackgroundJob\Legacy\RegularJob; +use OC\Core\Command\Background\Queue\Status; +use OCP\BackgroundJob\IJobList; +use Symfony\Component\Console\Application; +use Symfony\Component\Console\Tester\CommandTester; +use Test\TestCase; + +/** + * Class StatusTest + * + * @group DB + */ +class StatusTest extends TestCase { + + /** @var CommandTester */ + private $commandTester; + /** @var IJobList */ + private $jobList; + + public function setUp() { + parent::setUp(); + + $this->jobList = $this->createMock(IJobList::class); + $this->jobList->expects($this->any())->method('listJobs') + ->willReturnCallback(function (\Closure $callBack) { + $job = new RegularJob(); + $job->setId(666); + $callBack($job); + }); + + $command = new Status($this->jobList); + $command->setApplication(new Application()); + $this->commandTester = new CommandTester($command); + } + + public function testCommandInput() { + $this->commandTester->execute([]); + $output = $this->commandTester->getDisplay(); + $expected = <<assertContains($expected, $output); + } +}