feat(crontab): Add torrent status sync

Rhilip · Aug 6, 2019 · 1e600a0 · 1e600a0
1 parent 7497623
commit 1e600a0
Show file tree

Hide file tree

Showing 6 changed files with 90 additions and 45 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,11 +8,12 @@
 - **gitignore:** Add ignore of `/backup` folder
 
 ### Feat
-- **Category:** Add Image and class_name support
 - **Category:** Add Default sprite image of category
-- **Category:** Add Categories Manage Pane
+- **Category:** Add Image and class_name support
 - **Category:** Add Categories Support when upload torrent
+- **Category:** Add Categories Manage Pane
 - **Crontab:** Move From Timer to Process
+- **Editor:** Support wysibb editor
 - **Gravatar:** Add support of gravatar
 - **Process:** Add custom Process Support
 - **Redis:** Add mutiDelete() function for Redis

diff --git a/apps/libraries/Constant.php b/apps/libraries/Constant.php
@@ -38,13 +38,19 @@ public static function userContent($uid)
         return 'User:user_' . $uid . '_content:hash';
     }
 
+    public static function torrentContent($tid)
+    {
+        return 'Torrent:torrent_' . $tid . 'content:hash';
+    }
+
     // Tracker User
     public static function trackerUserContentByPasskey($passkey)
     {
         return 'Tracker:user_passkey_' . $passkey . '_content:string'; // Used string to store hash
     }
 
-    public static function trackerTorrentContentByInfoHash($bin2hex_hash){
+    public static function trackerTorrentContentByInfoHash($bin2hex_hash)
+    {
         return 'Tracker:torrent_infohash_' . $bin2hex_hash . '_content:hash';
     }
 

diff --git a/apps/models/Torrent.php b/apps/models/Torrent.php
@@ -8,6 +8,8 @@
 
 namespace apps\models;
 
+use apps\libraries\Constant;
+
 use Rid\Bencode\Bencode;
 use Rid\Exceptions\NotFoundException;
 use Rid\Utils\AttributesImportUtils;
@@ -59,26 +61,26 @@ public function __construct($id = null)
     {
         $this->loadTorrentContentById($id);
         if ($this->id == null) {
-            throw new NotFoundException("Not Found");
+            throw new NotFoundException('Not Found');  // FIXME
         }
     }
 
     public function loadTorrentContentById($id)
     {
-        $self = app()->redis->hGetAll('Torrent:' . $id . ':base_content');
+        $self = app()->redis->hGetAll(Constant::torrentContent($id));
         if (empty($self)) {
             $self = app()->pdo->createCommand("SELECT * FROM `torrents` WHERE id=:id LIMIT 1;")->bindParams([
                     "id" => $id
                 ])->queryOne() ?? [];
-            app()->redis->hMset('Torrent:' . $id . ':base_content', $self);
-            app()->redis->expire('Torrent:' . $id . ':base_content', 900);
+            app()->redis->hMset(Constant::torrentContent($id), $self);
+            app()->redis->expire(Constant::torrentContent($id), 1800);
         }
         $this->importAttributes($self);
     }
 
     public static function TorrentFileLoc($id = 0)
     {
-        return app()->getPrivatePath('torrents') . DIRECTORY_SEPARATOR . $id . ".torrent";
+        return app()->getPrivatePath('torrents') . DIRECTORY_SEPARATOR . $id . '.torrent';
     }
 
     /**
@@ -91,7 +93,7 @@ public function getTorrentStructure()
 
     protected function getCacheNameSpace(): string
     {
-        return 'Torrent:' . $this->id . ':base_content';
+        return Constant::torrentContent($this->id);
     }
 
     /**
@@ -187,26 +189,26 @@ public function getDownloadDict($encode = true)
     {
         $dict = $this->getRawDict();
 
-        $scheme = "http://";
-        if (filter_var(app()->request->get("https"), FILTER_VALIDATE_BOOLEAN))
-            $scheme = "https://";
-        else if (filter_var(app()->request->get("https"), FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE))
-            $scheme = "http://";
+        $scheme = 'http://';
+        if (filter_var(app()->request->get('https'), FILTER_VALIDATE_BOOLEAN))
+            $scheme = 'https://';
+        else if (filter_var(app()->request->get('https'), FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE))
+            $scheme = 'http://';
         else if (app()->request->isSecure())
-            $scheme = "https://";
+            $scheme = 'https://';
 
-        $announce_suffix = "/announce?passkey=" . app()->site->getCurUser()->getPasskey();
-        $dict["announce"] = $scheme . config("base.site_tracker_url") . $announce_suffix;
+        $announce_suffix = '/announce?passkey=' . app()->site->getCurUser()->getPasskey();
+        $dict['announce'] = $scheme . config('base.site_tracker_url') . $announce_suffix;
 
         /** BEP 0012 Multitracker Metadata Extension
          * @see http://www.bittorrent.org/beps/bep_0012.html
          * @see https://web.archive.org/web/20190724110959/https://blog.rhilip.info/archives/1108/
          *      which discuss about multitracker behaviour on common bittorrent client ( Chinese Version )
          */
-        if ($multi_trackers = config("base.site_multi_tracker_url")) {
+        if ($multi_trackers = config('base.site_multi_tracker_url')) {
             // Add our main tracker into multi_tracker_list to avoid lost....
-            $multi_trackers = config("base.site_tracker_url") . "," . $multi_trackers;
-            $multi_trackers_list = explode(",", $multi_trackers);
+            $multi_trackers = config('base.site_tracker_url') . ',' . $multi_trackers;
+            $multi_trackers_list = explode(',', $multi_trackers);
             $multi_trackers_list = array_unique($multi_trackers_list);  // use array_unique to remove dupe tracker
             // fulfill each tracker with scheme and suffix about user identity
             $multi_trackers_list = array_map(function ($uri) use ($scheme, $announce_suffix) {

diff --git a/apps/process/CronTabProcess.php b/apps/process/CronTabProcess.php
@@ -142,14 +142,46 @@ protected function expired_invitee()
     // sync torrents status about complete, incomplete, comments
     protected function sync_torrents_status()
     {
-        $cur_peer_status = app()->pdo->createCommand('SELECT `torrent_id`, `seeder`, COUNT(`id`) as `c` FROM `peers` GROUP BY `torrent_id`,`seeder`')->queryAll();
-        $cur_commit_status = app()->pdo->createCommand('SELECT `torrent_id`, COUNT(`id`) as `c` FROM `torrent_comments` GROUP BY `torrent_id`')->queryAll();
-        $cur_torrent_status = app()->pdo->createCommand('SELECT `id`,`complete`,`incomplete`,`comments` FROM `torrents`')->queryAll();
+        $torrents_update = [];
+
+        $wrong_complete_records = app()->pdo->createCommand("
+            SELECT torrents.`id`, `complete` AS `record`, COUNT(`peers`.id) AS `real` FROM `torrents` 
+              LEFT JOIN peers ON `peers`.torrent_id = `torrents`.id AND `peers`.`seeder` = 'yes' 
+            GROUP BY torrents.`id` HAVING `record` != `real`;")->queryAll();
+        if ($wrong_complete_records) {
+            array_walk($wrong_complete_records, function ($arr) use (&$torrents_update) {
+                $torrents_update[$arr['id']]['complete'] = $arr['real'];
+            });
+        }
+        $wrong_incomplete_records = app()->pdo->createCommand("
+            SELECT torrents.`id`, `incomplete` AS `record`, COUNT(`peers`.id) AS `real` FROM `torrents`
+              LEFT JOIN peers ON `peers`.torrent_id = `torrents`.id AND (`peers`.`seeder` = 'partial' OR `peers`.`seeder` = 'no') 
+            GROUP BY torrents.`id` HAVING `record` != `real`;")->queryAll();
+        if ($wrong_incomplete_records) {
+            array_walk($wrong_incomplete_records, function ($arr) use (&$torrents_update) {
+                $torrents_update[$arr['id']]['incomplete'] = $arr['real'];
+            });
+        }
 
+        $wrong_comment_records = app()->pdo->createCommand('
+            SELECT t.id, t.comments as `record`, COUNT(tc.id) as `real` FROM torrents t 
+              LEFT JOIN torrent_comments tc on t.id = tc.torrent_id 
+            GROUP BY t.id HAVING `record` != `real`')->queryAll();
+        if ($wrong_comment_records) {
+            array_walk($wrong_comment_records, function ($arr) use (&$torrents_update) {
+                $torrents_update[$arr['id']]['comments'] = $arr['real'];
+            });
+        }
 
+        if ($torrents_update) {
+            foreach ($torrents_update as $tid => $update) {
+                app()->pdo->update('torrents', $update, [['id', '=', $tid]])->execute();
+                app()->redis->del(Constant::torrentContent($tid));
+            }
+            $this->print_log('Fix ' . count($torrents_update) . ' wrong torrents records about complete, incomplete, comments.');
+        }
     }
 
-
     protected function update_expired_external_link_info()
     {
         $expired_links_res = app()->pdo->createCommand('SELECT `source`,`sid` FROM `external_info` ORDER BY `update_at` ASC LIMIt 5')->queryAll();

diff --git a/apps/process/TrackerAnnounceProcess.php b/apps/process/TrackerAnnounceProcess.php
@@ -15,27 +15,30 @@ class TrackerAnnounceProcess extends Process
 {
     public function run()
     {
-        $data = app()->redis->brpoplpush(Constant::trackerToDealQueue, Constant::trackerBackupQueue, 5);
-        if ($data !== false) {
-            app()->pdo->beginTransaction();
-            try {
-                /* We got data from Http Server Like
-                 * [
-                 *    'timestamp' => timestamp when controller receive the announce,
-                 *    'queries' => $queries, 'role' => $role,
-                 *    'userInfo' => $userInfo, 'torrentInfo' => $torrentInfo
-                 * ]
-                 */
-                $this->processAnnounceRequest($data['timestamp'], $data['queries'], $data['role'], $data['userInfo'], $data['torrentInfo']);
+        do {
+            $data = app()->redis->brpoplpush(Constant::trackerToDealQueue, Constant::trackerBackupQueue, 5);
+            if ($data !== false) {
+                app()->pdo->beginTransaction();
+                try {
+                    /* We got data from Http Server Like
+                     * [
+                     *    'timestamp' => timestamp when controller receive the announce,
+                     *    'queries' => $queries, 'role' => $role,
+                     *    'userInfo' => $userInfo, 'torrentInfo' => $torrentInfo
+                     * ]
+                     */
+                    $this->processAnnounceRequest($data['timestamp'], $data['queries'], $data['role'], $data['userInfo'], $data['torrentInfo']);
 
-                app()->pdo->commit();
-                app()->redis->lRem(Constant::trackerBackupQueue, $data, 0);
-            } catch (\Exception $e) {
-                println($e->getMessage());
-                app()->pdo->rollback();
-                // TODO deal with the items in backup_queue
+                    app()->pdo->commit();
+                    app()->redis->lRem(Constant::trackerBackupQueue, $data, 0);
+                } catch (\Exception $e) {
+                    println($e->getMessage());
+                    app()->pdo->rollback();
+                    // TODO deal with the items in backup_queue
+                }
             }
-        }
+            \Rid::app()->cleanComponents();
+        } while ($data !== false);
     }
 
     /**

diff --git a/migration/ridpt.sql b/migration/ridpt.sql
@@ -3,7 +3,7 @@
 -- https://www.phpmyadmin.net/
 --
 -- Host: 127.0.0.1
--- Generation Time: Aug 01, 2019 at 10:38 PM
+-- Generation Time: Aug 06, 2019 at 03:36 PM
 -- Server version: 8.0.16
 -- PHP Version: 7.3.7
 
@@ -833,7 +833,8 @@ INSERT INTO `site_crontab` (`id`, `job`, `priority`, `job_interval`) VALUES
 (3, 'calculate_seeding_bonus', 2, 900),
 (4, 'clean_expired_session', 3, 600),
 (5, 'expired_invitee', 3, 600),
-(6, 'update_expired_external_link_info', 100, 1200);
+(6, 'update_expired_external_link_info', 100, 1200),
+(7, 'sync_torrents_status', 4, 3600);
 
 -- --------------------------------------------------------