Improved performance of journal cleaning

src/Kdyby/Redis/RedisJournal.php

@@ -31,6 +31,9 @@ class RedisJournal extends Nette\Object implements Nette\Caching\Storages\IJourn
  TAGS = 'tags',
  KEYS = 'keys';
 
+ /** @internal batch delete size */
+ const BATCH_SIZE = 8000;
+
  /**
  * @var RedisClient
  */
@@ -108,34 +111,86 @@ private function cleanEntry($keys)
  * Cleans entries from journal.
  *
  * @param array $conds
+ * @param \Nette\Caching\IStorage $storage
  *
  * @return array of removed items or NULL when performing a full cleanup
  */
- public function clean(array $conds)
+ public function clean(array $conds, Nette\Caching\IStorage $storage = NULL)
  {
  if (!empty($conds[Cache::ALL])) {
  $all = $this->client->keys(self::NS_NETTE . ':*');
+ if ($storage instanceof RedisStorage) {
+ $all = array_merge($all, $this->client->keys(RedisStorage::NS_NETTE . ':*'));
+ }
 
- $this->client->multi();
  call_user_func_array(array($this->client, 'del'), $all);
- $this->client->exec();
  return NULL;
  }
 
  $entries = array();
  if (!empty($conds[Cache::TAGS])) {
+ $removingTagKeys = array();
  foreach ((array)$conds[Cache::TAGS] as $tag) {
- $this->cleanEntry($found = $this->tagEntries($tag));
+ $found = $this->tagEntries($tag);
+ $removingTagKeys[] = $this->formatKey($tag, self::KEYS);
  $entries = array_merge($entries, $found);
  }
+ if ($removingTagKeys) {
+ call_user_func_array(array($this->client, 'del'), $removingTagKeys);
+ }
  }
 
  if (isset($conds[Cache::PRIORITY])) {
- $this->cleanEntry($found = $this->priorityEntries($conds[Cache::PRIORITY]));
+ $found = $this->priorityEntries($conds[Cache::PRIORITY]);
+ call_user_func_array(array($this->client, 'zRemRangeByScore'), array($this->formatKey(self::PRIORITY), 0, (int)$conds[Cache::PRIORITY]));
  $entries = array_merge($entries, $found);
  }
 
- return array_unique($entries);
+ $entries = array_unique($entries);
+
+ $removingKeys = array();
+ $removingKeyTags = array();
+ $removingKeyPriorities = array();
+ foreach ($entries as $key) {
+ if ($storage instanceof RedisStorage) {
+ $removingKeys[] = $key;
+ }
+ $removingKeyTags[] = $this->formatKey($key, self::TAGS);
+ $removingKeyPriorities[] = $this->formatKey($key, self::PRIORITY);
+ if (count($removingKeyTags) >= self::BATCH_SIZE) {
+ $this->cleanBatchData($removingKeys, $removingKeyPriorities, $removingKeyTags, $entries);
+ $removingKeys = array();
+ $removingKeyTags = array();
+ $removingKeyPriorities = array();
+ }
+ }
+
+ $this->cleanBatchData($removingKeys, $removingKeyPriorities, $removingKeyTags, $entries);
+
+ return $storage instanceof RedisStorage ? array() : $entries;
+ }
+
+
+
+ private function cleanBatchData(array $removingKeys, array $removingKeyPriorities, array $removingKeyTags, array $keys)
+ {
+ if ($removingKeyTags) {
+ if ($keys) {
+ $affectedTags = call_user_func_array(array($this->client, 'sunion'), array($removingKeyTags));
+ foreach ($affectedTags as $tag) {
+ if ($tag) {
+ call_user_func_array(array($this->client, 'sRem'), array_merge(array($this->formatKey($tag, self::KEYS)), $keys));
+ }
+ }
+ }
+ call_user_func_array(array($this->client, 'del'), $removingKeyTags);
+ }
+ if ($removingKeyPriorities) {
+ call_user_func_array(array($this->client, 'del'), $removingKeyPriorities);
+ }
+ if ($removingKeys) {
+ call_user_func_array(array($this->client, 'del'), $removingKeys);
+ }
  }
 
 

src/Kdyby/Redis/scripts/common.lua

@@ -29,6 +29,23 @@ local tagEntries = function (tag)
  return redis.call('sMembers', formatKey(tag, "keys"))
 end
 
+local range = function (from, to, step)
+ step = step or 1
+ local f =
+ step > 0 and
+ function(_, lastvalue)
+ local nextvalue = lastvalue + step
+ if nextvalue <= to then return nextvalue end
+ end or
+ step < 0 and
+ function(_, lastvalue)
+ local nextvalue = lastvalue + step
+ if nextvalue >= to then return nextvalue end
+ end or
+ function(_, lastvalue) return lastvalue end
+ return f, nil, from - step
+end
+
 local cleanEntry = function (keys)
  for i, key in pairs(keys) do
  local tags = entryTags(key)
@@ -45,3 +62,34 @@ local cleanEntry = function (keys)
  -- redis.call('exec')
  end
 end
+
+local mergeTables = function (first, second)
+ for i, key in pairs(second) do
+ first[#first + 1] = key
+ end
+ return first
+end
+
+local batch = function (keys, callback)
+ if #keys > 0 then
+ -- redis.call('multi')
+ -- the magic number 7998 becomes from Lua limitations, see http://stackoverflow.com/questions/19202367/how-to-avoid-redis-calls-in-lua-script-limitations
+ local tmp = {}
+ for i,key in pairs(keys) do
+ tmp[#tmp + 1] = key
+ if #tmp >= 7998 then
+ callback(tmp)
+ tmp = {}
+ end
+ end
+ callback(tmp)
+ -- redis.call('exec')
+ end
+end
+
+local batchDelete = function(keys)
+ local delete = function (tmp)
+ redis.call('del', unpack(tmp))
+ end
+ batch(keys, delete)
+end

src/Kdyby/Redis/scripts/journal.clean.lua

@@ -2,11 +2,10 @@
 local conds = cjson.decode(ARGV[1])
 
 if conds["all"] ~= nil then
- -- redis.call('multi')
- for i, value in pairs(redis.call('keys', "Nette.Journal:*")) do
- redis.call('del', value)
+ batchDelete(redis.call('keys', "Nette.Journal:*"))
+ if conds["delete-entries"] ~= nil then
+ batchDelete(redis.call('keys', "Nette.Storage:*"))
  end
- -- redis.call('exec')
 
  return redis.status_reply("Ok")
 end

tests/KdybyTests/Redis/RedisJournal.phpt

@@ -284,7 +284,7 @@ class RedisJournalTest extends AbstractRedisTestCase
  Assert::null($result);
 
  $result2 = $this->journal->clean(array(Cache::TAGS => 'test:all'));
- Assert::true(empty($result2));
+ Assert::equal(array(), $result2);
  }
 
 
@@ -362,26 +362,7 @@ LUA;
  private function cacheGeneratorScripts()
  {
  $script = file_get_contents(__DIR__ . '/../../../src/Kdyby/Redis/scripts/common.lua');
- $script .= <<<LUA
-local range = function (from, to, step)
- step = step or 1
- local f =
- step > 0 and
- function(_, lastvalue)
- local nextvalue = lastvalue + step
- if nextvalue <= to then return nextvalue end
- end or
- step < 0 and
- function(_, lastvalue)
- local nextvalue = lastvalue + step
- if nextvalue >= to then return nextvalue end
- end or
- function(_, lastvalue) return lastvalue end
- return f, nil, from - step
-end
 
-
-LUA;
  return $script;
  }