feat(ban): Sync site ban list of username and email in Crontab Jobs

Sync site ban list of username and email in Crontab Jobs, so we can quick check username or email is in our blacklist or not by call Redis::sIsMember()

CHANGELOG.md

@@ -41,15 +41,16 @@
 - **Process:** Disable Pdo And Redis called data in custom process (b744e81)
 
 ### Refactor
+- **Array:** Move setDefault for Array as global function array_set_default (b825eca)
 - **Auth:** Fix Certification process (687a2d0)
 - **Auth/JWT:** Better for auth by JWT (36f49a0)
 - **Auth/Middleware:** merge Old Auth{ByCookies, ByPasskey}Middleware (71cd7d7)
-- **Config:** Remove params `$throw` in Config()->get() (706cc9a)
 - **Config:** Add define of config key type and can add runtime config (d57aede)
+- **Config:** Remove params `$throw` in Config()->get() (706cc9a)
 - **Controller:** Move APIController out Framework (0dc7106)
 - **RateLimit:** Change last param of isRateLimitHit and rate limit store Namespace (4dd571d)
-- **Site:** Move Cat, Quality, PinnedTag cache to Config.runtime (da1d9a7)
 - **Site:** Simple Category Detail get function (ffa6855)
+- **Site:** Move Cat, Quality, PinnedTag cache to Config.runtime (da1d9a7)
 - **Validator:** fix user input extract (81bdc8f)
 - **View:** Make View extends BaseObject (0865cf9)
 - **action:** Sort template action/action_{fail,success} (66998d3)

framework/Base/Process.php

@@ -17,6 +17,11 @@ class Process implements StaticInstanceInterface
  private $sleep_time;
  protected $_config;
 
+ public function init()
+ {
+
+ }
+
  public function run()
  {
 
@@ -51,6 +56,7 @@ final public function start($config)
 
  println('New Custom process `' . static::class . '` added.');
 
+ $this->init();
  while (true) {
  $this->run();
  sleep($this->getSleepTime());

migration/ridpt.sql

@@ -3,7 +3,7 @@
 -- https://www.phpmyadmin.net/
 --
 -- Host: 127.0.0.1
--- Generation Time: Aug 26, 2019 at 10:10 PM
+-- Generation Time: Sep 13, 2019 at 04:39 PM
 -- Server version: 8.0.17
 -- PHP Version: 7.3.7
 
@@ -921,7 +921,8 @@ INSERT INTO `site_crontab` (`id`, `job`, `priority`, `job_interval`) VALUES
 (3, 'clean_expired_items_database', 3, 3600),
 (4, 'calculate_seeding_bonus', 2, 900),
 (5, 'sync_torrents_status', 4, 3600),
-(6, 'update_expired_external_link_info', 100, 1200);
+(6, 'update_expired_external_link_info', 100, 1200),
+(7, 'sync_ban_list', 100, 86400);
 
 -- --------------------------------------------------------
 

src/Libraries/Constant.php

@@ -33,6 +33,8 @@ class Constant
 
  // Site Status
  const siteSubtitleSize = 'Site:subtitle_size'; // TODO move to app()->config
+ const siteBannedEmailSet = 'Site:set:banned_list:email';
+ const siteBannedUsernameSet = 'Site:set:banned_list:username';
 
  public static function userContent(int $uid)
  {

src/Models/Form/Auth/UserRegisterForm.php

@@ -8,6 +8,7 @@
 
 namespace App\Models\Form\Auth;
 
+use App\Libraries\Constant;
 use App\Models\User;
 
 use Rid\Helpers\StringHelper;
@@ -145,7 +146,7 @@ protected function isMaxRegisterIpReached()
  $client_ip = app()->request->getClientIp();
 
  $max_user_per_ip = config('register.per_ip_user') ?: 5;
- $user_ip_count = app()->pdo->createCommand("SELECT COUNT(`id`) FROM `users` WHERE `register_ip` = INET6_ATON(:ip)")->bindParams([
+ $user_ip_count = app()->pdo->createCommand('SELECT COUNT(`id`) FROM `users` WHERE `register_ip` = INET6_ATON(:ip)')->bindParams([
  "ip" => $client_ip
  ])->queryScalar();
 
@@ -165,19 +166,14 @@ protected function isValidUsername()
  return;
  }
 
- // Check if this username is not in blacklist
- if (!app()->redis->exists('site:username_ban_list')) {
- $ban_username_list = app()->pdo->createCommand('SELECT `username` from `ban_usernames`')->queryColumn() ?: [];
- app()->redis->hMset('site:username_ban_list', $ban_username_list);
- app()->redis->expire('site:username_ban_list', 86400);
- }
- if (app()->redis->hExists('site:username_ban_list', $username)) {
+ // Check if this username is in blacklist or not
+ if (app()->redis->sIsMember(Constant::siteBannedUsernameSet, $username)) {
  $this->buildCallbackFailMsg('ValidUsername', 'This username is in our blacklist.');
  return;
  }
 
  // Check this username is exist in Table `users` or not
- $count = app()->pdo->createCommand("SELECT COUNT(`id`) FROM `users` WHERE `username` = :username")->bindParams([
+ $count = app()->pdo->createCommand('SELECT COUNT(`id`) FROM `users` WHERE `username` = :username')->bindParams([
  'username' => $username
  ])->queryScalar();
  if ($count > 0) {
@@ -192,7 +188,7 @@ protected function isValidEmail()
  $email_suffix = substr($email, strpos($email, '@')); // Will get `@test.com` as example
  if (config('register.check_email_blacklist') &&
  config('register.email_black_list')) {
- $email_black_list = explode(",", config('register.email_black_list'));
+ $email_black_list = explode(',', config('register.email_black_list'));
  if (in_array($email_suffix, $email_black_list)) {
  $this->buildCallbackFailMsg('ValidEmail', "The email suffix `$email_suffix` is not allowed.");
  return;
@@ -201,25 +197,20 @@ protected function isValidEmail()
 
  if (config('register.check_email_whitelist') &&
  config('register.email_white_list')) {
- $email_white_list = explode(",", config('register.email_white_list'));
+ $email_white_list = explode(',', config('register.email_white_list'));
  if (!in_array($email_suffix, $email_white_list)) {
  $this->buildCallbackFailMsg('ValidEmail', "The email suffix `$email_suffix` is not allowed.");
  return;
  }
  }
 
- // Check $email is not in blacklist
- if (!app()->redis->exists('site:emails_ban_list')) {
- $ban_email_list = app()->pdo->createCommand('SELECT `email` from `ban_emails`')->queryColumn();
- app()->redis->hMset('site:emails_ban_list', $ban_email_list);
- app()->redis->expire('site:emails_ban_list', 86400);
- }
- if (app()->redis->hExists('site:emails_ban_list', $email)) {
+ // Check $email is in blacklist or not
+ if (app()->redis->sIsMember(Constant::siteBannedEmailSet, $email)) {
  $this->buildCallbackFailMsg('ValidEmail', 'This email is in our blacklist.');
  return;
  }
 
- $email_check = app()->pdo->createCommand("SELECT COUNT(`id`) FROM `users` WHERE `email` = :email")->bindParams([
+ $email_check = app()->pdo->createCommand('SELECT COUNT(`id`) FROM `users` WHERE `email` = :email')->bindParams([
  "email" => $email
  ])->queryScalar();
  if ($email_check > 0) {

src/Process/CronTabProcess.php

@@ -10,6 +10,7 @@
 
 use App\Libraries\Bonus;
 use App\Libraries\Constant;
+
 use Rid\Base\Process;
 
 
@@ -65,8 +66,8 @@ public function run()
  app()->log->critical('The run job throw Exception : ' . $e->getMessage());
  }
  } else {
- if (!in_array($job, $this->_none_exist_job)) {
- $this->_none_exist_job[] = $job;
+ if (!in_array($job['job'], $this->_none_exist_job)) {
+ $this->_none_exist_job[] = $job['job'];
  app()->log->critical('CronTab Worker Tries to run a none-exist job:' . $job['job']);
  }
  }
@@ -143,7 +144,10 @@ protected function calculate_seeding_bonus() // TODO
  }
  }
 
- // sync torrents status about complete, incomplete, comments
+ /**
+ * sync torrents status about complete, incomplete, comments
+ * @noinspection PhpUnused
+ */
  protected function sync_torrents_status()
  {
  $torrents_update = [];
@@ -153,28 +157,28 @@ protected function sync_torrents_status()
  LEFT JOIN peers ON `peers`.torrent_id = `torrents`.id AND `peers`.`seeder` = 'yes' 
  GROUP BY torrents.`id` HAVING `record` != `real`;")->queryAll();
  if ($wrong_complete_records) {
- array_walk($wrong_complete_records, function ($arr) use (&$torrents_update) {
+ foreach ($wrong_complete_records as $arr) {
  $torrents_update[$arr['id']]['complete'] = $arr['real'];
- });
+ }
  }
  $wrong_incomplete_records = app()->pdo->createCommand("
  SELECT torrents.`id`, `incomplete` AS `record`, COUNT(`peers`.id) AS `real` FROM `torrents`
  LEFT JOIN peers ON `peers`.torrent_id = `torrents`.id AND (`peers`.`seeder` = 'partial' OR `peers`.`seeder` = 'no') 
  GROUP BY torrents.`id` HAVING `record` != `real`;")->queryAll();
  if ($wrong_incomplete_records) {
- array_walk($wrong_incomplete_records, function ($arr) use (&$torrents_update) {
+ foreach ($wrong_incomplete_records as $arr) {
  $torrents_update[$arr['id']]['incomplete'] = $arr['real'];
- });
+ }
  }
 
  $wrong_comment_records = app()->pdo->createCommand('
  SELECT t.id, t.comments as `record`, COUNT(tc.id) as `real` FROM torrents t 
  LEFT JOIN torrent_comments tc on t.id = tc.torrent_id 
  GROUP BY t.id HAVING `record` != `real`')->queryAll();
  if ($wrong_comment_records) {
- array_walk($wrong_comment_records, function ($arr) use (&$torrents_update) {
+ foreach ($wrong_incomplete_records as $arr) {
  $torrents_update[$arr['id']]['comments'] = $arr['real'];
- });
+ }
  }
 
  if ($torrents_update) {
@@ -186,6 +190,17 @@ protected function sync_torrents_status()
  }
  }
 
+ /** @noinspection PhpUnused */
+ protected function sync_ban_list() {
+ // Sync Banned Emails list
+ $ban_email_list = app()->pdo->createCommand('SELECT `email` from `ban_emails`')->queryColumn() ?: [];
+ app()->redis->sAddArray(Constant::siteBannedEmailSet, $ban_email_list);
+
+ // Sync Banned Username list
+ $ban_username_list = app()->pdo->createCommand('SELECT `username` from `ban_usernames`')->queryColumn() ?: [];
+ app()->redis->sAddArray(Constant::siteBannedUsernameSet, $ban_username_list);
+ }
+
  protected function update_expired_external_link_info()
  {
  $expired_links_res = app()->pdo->createCommand('SELECT `source`,`sid` FROM `external_info` ORDER BY `update_at` ASC LIMIt 5')->queryAll();