From 33cc1e64f108731afed6dbd5f5a6d0717e455470 Mon Sep 17 00:00:00 2001 From: Rhilip Date: Fri, 13 Sep 2019 16:43:09 +0800 Subject: [PATCH] feat(ban): Sync site ban list of username and email in Crontab Jobs Sync site ban list of username and email in Crontab Jobs, so we can quick check username or email is in our blacklist or not by call Redis::sIsMember() --- CHANGELOG.md | 5 ++-- framework/Base/Process.php | 6 +++++ migration/ridpt.sql | 5 ++-- src/Libraries/Constant.php | 2 ++ src/Models/Form/Auth/UserRegisterForm.php | 29 +++++++------------- src/Process/CronTabProcess.php | 33 ++++++++++++++++------- 6 files changed, 48 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c965f7..dbaab8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,15 +41,16 @@ - **Process:** Disable Pdo And Redis called data in custom process (b744e81) ### Refactor +- **Array:** Move setDefault for Array as global function array_set_default (b825eca) - **Auth:** Fix Certification process (687a2d0) - **Auth/JWT:** Better for auth by JWT (36f49a0) - **Auth/Middleware:** merge Old Auth{ByCookies, ByPasskey}Middleware (71cd7d7) -- **Config:** Remove params `$throw` in Config()->get() (706cc9a) - **Config:** Add define of config key type and can add runtime config (d57aede) +- **Config:** Remove params `$throw` in Config()->get() (706cc9a) - **Controller:** Move APIController out Framework (0dc7106) - **RateLimit:** Change last param of isRateLimitHit and rate limit store Namespace (4dd571d) -- **Site:** Move Cat, Quality, PinnedTag cache to Config.runtime (da1d9a7) - **Site:** Simple Category Detail get function (ffa6855) +- **Site:** Move Cat, Quality, PinnedTag cache to Config.runtime (da1d9a7) - **Validator:** fix user input extract (81bdc8f) - **View:** Make View extends BaseObject (0865cf9) - **action:** Sort template action/action_{fail,success} (66998d3) diff --git a/framework/Base/Process.php b/framework/Base/Process.php index 4ef6c04..5f15005 100644 --- a/framework/Base/Process.php +++ b/framework/Base/Process.php @@ -17,6 +17,11 @@ class Process implements StaticInstanceInterface private $sleep_time; protected $_config; + public function init() + { + + } + public function run() { @@ -51,6 +56,7 @@ final public function start($config) println('New Custom process `' . static::class . '` added.'); + $this->init(); while (true) { $this->run(); sleep($this->getSleepTime()); diff --git a/migration/ridpt.sql b/migration/ridpt.sql index bcd58ba..adf13e4 100644 --- a/migration/ridpt.sql +++ b/migration/ridpt.sql @@ -3,7 +3,7 @@ -- https://www.phpmyadmin.net/ -- -- Host: 127.0.0.1 --- Generation Time: Aug 26, 2019 at 10:10 PM +-- Generation Time: Sep 13, 2019 at 04:39 PM -- Server version: 8.0.17 -- PHP Version: 7.3.7 @@ -921,7 +921,8 @@ INSERT INTO `site_crontab` (`id`, `job`, `priority`, `job_interval`) VALUES (3, 'clean_expired_items_database', 3, 3600), (4, 'calculate_seeding_bonus', 2, 900), (5, 'sync_torrents_status', 4, 3600), -(6, 'update_expired_external_link_info', 100, 1200); +(6, 'update_expired_external_link_info', 100, 1200), +(7, 'sync_ban_list', 100, 86400); -- -------------------------------------------------------- diff --git a/src/Libraries/Constant.php b/src/Libraries/Constant.php index 9a962fd..2f17d27 100644 --- a/src/Libraries/Constant.php +++ b/src/Libraries/Constant.php @@ -33,6 +33,8 @@ class Constant // Site Status const siteSubtitleSize = 'Site:subtitle_size'; // TODO move to app()->config + const siteBannedEmailSet = 'Site:set:banned_list:email'; + const siteBannedUsernameSet = 'Site:set:banned_list:username'; public static function userContent(int $uid) { diff --git a/src/Models/Form/Auth/UserRegisterForm.php b/src/Models/Form/Auth/UserRegisterForm.php index 62464f6..36238d1 100644 --- a/src/Models/Form/Auth/UserRegisterForm.php +++ b/src/Models/Form/Auth/UserRegisterForm.php @@ -8,6 +8,7 @@ namespace App\Models\Form\Auth; +use App\Libraries\Constant; use App\Models\User; use Rid\Helpers\StringHelper; @@ -145,7 +146,7 @@ protected function isMaxRegisterIpReached() $client_ip = app()->request->getClientIp(); $max_user_per_ip = config('register.per_ip_user') ?: 5; - $user_ip_count = app()->pdo->createCommand("SELECT COUNT(`id`) FROM `users` WHERE `register_ip` = INET6_ATON(:ip)")->bindParams([ + $user_ip_count = app()->pdo->createCommand('SELECT COUNT(`id`) FROM `users` WHERE `register_ip` = INET6_ATON(:ip)')->bindParams([ "ip" => $client_ip ])->queryScalar(); @@ -165,19 +166,14 @@ protected function isValidUsername() return; } - // Check if this username is not in blacklist - if (!app()->redis->exists('site:username_ban_list')) { - $ban_username_list = app()->pdo->createCommand('SELECT `username` from `ban_usernames`')->queryColumn() ?: []; - app()->redis->hMset('site:username_ban_list', $ban_username_list); - app()->redis->expire('site:username_ban_list', 86400); - } - if (app()->redis->hExists('site:username_ban_list', $username)) { + // Check if this username is in blacklist or not + if (app()->redis->sIsMember(Constant::siteBannedUsernameSet, $username)) { $this->buildCallbackFailMsg('ValidUsername', 'This username is in our blacklist.'); return; } // Check this username is exist in Table `users` or not - $count = app()->pdo->createCommand("SELECT COUNT(`id`) FROM `users` WHERE `username` = :username")->bindParams([ + $count = app()->pdo->createCommand('SELECT COUNT(`id`) FROM `users` WHERE `username` = :username')->bindParams([ 'username' => $username ])->queryScalar(); if ($count > 0) { @@ -192,7 +188,7 @@ protected function isValidEmail() $email_suffix = substr($email, strpos($email, '@')); // Will get `@test.com` as example if (config('register.check_email_blacklist') && config('register.email_black_list')) { - $email_black_list = explode(",", config('register.email_black_list')); + $email_black_list = explode(',', config('register.email_black_list')); if (in_array($email_suffix, $email_black_list)) { $this->buildCallbackFailMsg('ValidEmail', "The email suffix `$email_suffix` is not allowed."); return; @@ -201,25 +197,20 @@ protected function isValidEmail() if (config('register.check_email_whitelist') && config('register.email_white_list')) { - $email_white_list = explode(",", config('register.email_white_list')); + $email_white_list = explode(',', config('register.email_white_list')); if (!in_array($email_suffix, $email_white_list)) { $this->buildCallbackFailMsg('ValidEmail', "The email suffix `$email_suffix` is not allowed."); return; } } - // Check $email is not in blacklist - if (!app()->redis->exists('site:emails_ban_list')) { - $ban_email_list = app()->pdo->createCommand('SELECT `email` from `ban_emails`')->queryColumn(); - app()->redis->hMset('site:emails_ban_list', $ban_email_list); - app()->redis->expire('site:emails_ban_list', 86400); - } - if (app()->redis->hExists('site:emails_ban_list', $email)) { + // Check $email is in blacklist or not + if (app()->redis->sIsMember(Constant::siteBannedEmailSet, $email)) { $this->buildCallbackFailMsg('ValidEmail', 'This email is in our blacklist.'); return; } - $email_check = app()->pdo->createCommand("SELECT COUNT(`id`) FROM `users` WHERE `email` = :email")->bindParams([ + $email_check = app()->pdo->createCommand('SELECT COUNT(`id`) FROM `users` WHERE `email` = :email')->bindParams([ "email" => $email ])->queryScalar(); if ($email_check > 0) { diff --git a/src/Process/CronTabProcess.php b/src/Process/CronTabProcess.php index 806b15b..2c6e0a8 100644 --- a/src/Process/CronTabProcess.php +++ b/src/Process/CronTabProcess.php @@ -10,6 +10,7 @@ use App\Libraries\Bonus; use App\Libraries\Constant; + use Rid\Base\Process; @@ -65,8 +66,8 @@ public function run() app()->log->critical('The run job throw Exception : ' . $e->getMessage()); } } else { - if (!in_array($job, $this->_none_exist_job)) { - $this->_none_exist_job[] = $job; + if (!in_array($job['job'], $this->_none_exist_job)) { + $this->_none_exist_job[] = $job['job']; app()->log->critical('CronTab Worker Tries to run a none-exist job:' . $job['job']); } } @@ -143,7 +144,10 @@ protected function calculate_seeding_bonus() // TODO } } - // sync torrents status about complete, incomplete, comments + /** + * sync torrents status about complete, incomplete, comments + * @noinspection PhpUnused + */ protected function sync_torrents_status() { $torrents_update = []; @@ -153,18 +157,18 @@ protected function sync_torrents_status() LEFT JOIN peers ON `peers`.torrent_id = `torrents`.id AND `peers`.`seeder` = 'yes' GROUP BY torrents.`id` HAVING `record` != `real`;")->queryAll(); if ($wrong_complete_records) { - array_walk($wrong_complete_records, function ($arr) use (&$torrents_update) { + foreach ($wrong_complete_records as $arr) { $torrents_update[$arr['id']]['complete'] = $arr['real']; - }); + } } $wrong_incomplete_records = app()->pdo->createCommand(" SELECT torrents.`id`, `incomplete` AS `record`, COUNT(`peers`.id) AS `real` FROM `torrents` LEFT JOIN peers ON `peers`.torrent_id = `torrents`.id AND (`peers`.`seeder` = 'partial' OR `peers`.`seeder` = 'no') GROUP BY torrents.`id` HAVING `record` != `real`;")->queryAll(); if ($wrong_incomplete_records) { - array_walk($wrong_incomplete_records, function ($arr) use (&$torrents_update) { + foreach ($wrong_incomplete_records as $arr) { $torrents_update[$arr['id']]['incomplete'] = $arr['real']; - }); + } } $wrong_comment_records = app()->pdo->createCommand(' @@ -172,9 +176,9 @@ protected function sync_torrents_status() LEFT JOIN torrent_comments tc on t.id = tc.torrent_id GROUP BY t.id HAVING `record` != `real`')->queryAll(); if ($wrong_comment_records) { - array_walk($wrong_comment_records, function ($arr) use (&$torrents_update) { + foreach ($wrong_incomplete_records as $arr) { $torrents_update[$arr['id']]['comments'] = $arr['real']; - }); + } } if ($torrents_update) { @@ -186,6 +190,17 @@ protected function sync_torrents_status() } } + /** @noinspection PhpUnused */ + protected function sync_ban_list() { + // Sync Banned Emails list + $ban_email_list = app()->pdo->createCommand('SELECT `email` from `ban_emails`')->queryColumn() ?: []; + app()->redis->sAddArray(Constant::siteBannedEmailSet, $ban_email_list); + + // Sync Banned Username list + $ban_username_list = app()->pdo->createCommand('SELECT `username` from `ban_usernames`')->queryColumn() ?: []; + app()->redis->sAddArray(Constant::siteBannedUsernameSet, $ban_username_list); + } + protected function update_expired_external_link_info() { $expired_links_res = app()->pdo->createCommand('SELECT `source`,`sid` FROM `external_info` ORDER BY `update_at` ASC LIMIt 5')->queryAll();